2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1999 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
26 /* $Id: chunker.c,v 1.36 2006/08/24 11:23:32 martinea Exp $
28 * requests remote amandad processes to dump filesystems
41 #include "fileheader.h"
42 #include "amfeatures.h"
43 #include "server_util.h"
46 #include "timestamp.h"
48 #define chunker_debug(i, ...) do { \
49 if ((i) <= debug_chunker) { \
50 dbprintf(__VA_ARGS__); \
62 #define CONNECT_TIMEOUT 5*60
64 #define STARTUP_TIMEOUT 60
67 int fd; /* file to flush to */
68 char *filename; /* name of what fd points to */
69 int filename_seq; /* for chunking */
70 off_t split_size; /* when to chunk */
71 off_t chunk_size; /* size of each chunk */
72 off_t use; /* size to use on this disk */
73 char buf[DISK_BLOCK_BYTES];
74 char *datain; /* data buffer markers */
79 static char *handle = NULL;
81 static char *errstr = NULL;
82 static int abort_pending;
83 static off_t dumpsize;
84 static unsigned long headersize;
85 static off_t dumpbytes;
86 static off_t filesize;
88 static char *hostname = NULL;
89 static char *diskname = NULL;
90 static char *qdiskname = NULL;
91 static char *options = NULL;
92 static char *progname = NULL;
94 static char *dumpdate = NULL;
95 static struct cmdargs *command_in_transit = NULL;
96 static char *chunker_timestamp = NULL;
98 static dumpfile_t file;
100 /* local functions */
101 int main(int, char **);
102 static ssize_t write_tapeheader(int, dumpfile_t *);
103 static void databuf_init(struct databuf *, int, char *, off_t, off_t);
104 static int databuf_flush(struct databuf *);
106 static int startup_chunker(char *, off_t, off_t, struct databuf *);
107 static int do_chunk(int, struct databuf *);
115 static struct databuf db;
116 struct cmdargs *cmdargs;
119 char *filename = NULL;
120 off_t chunksize, use;
122 am_feature_t *their_features = NULL;
124 config_overwrites_t *cfg_ovr = NULL;
125 char *cfg_opt = NULL;
129 * Configure program for internationalization:
130 * 1) Only set the message locale for now.
131 * 2) Set textdomain for all amanda related programs to "amanda"
132 * We don't want to be forced to support dozens of message catalogs.
134 setlocale(LC_MESSAGES, "C");
135 textdomain("amanda");
139 set_pname("chunker");
141 dbopen(DBG_SUBDIR_SERVER);
143 /* Don't die when child closes pipe */
144 signal(SIGPIPE, SIG_IGN);
146 erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE);
147 set_logerror(logerror);
149 cfg_ovr = extract_commandline_config_overwrites(&argc, &argv);
154 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
155 apply_config_overwrites(cfg_ovr);
157 if (config_errors(NULL) >= CFGERR_WARNINGS) {
158 config_print_errors();
159 if (config_errors(NULL) >= CFGERR_ERRORS) {
160 g_critical(_("errors processing config file"));
164 safe_cd(); /* do this *after* config_init() */
166 check_running_as(RUNNING_AS_DUMPUSER);
168 dbrename(get_config_name(), DBG_SUBDIR_SERVER);
170 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
172 _("%s: pid %ld executable %s version %s\n"),
173 get_pname(), (long) getpid(),
177 /* now, make sure we are a valid user */
179 signal(SIGPIPE, SIG_IGN);
180 signal(SIGCHLD, SIG_IGN);
183 if(cmdargs->cmd == START) {
184 if(cmdargs->argc <= 1)
185 error(_("error [dumper START: not enough args: timestamp]"));
186 chunker_timestamp = newstralloc(chunker_timestamp, cmdargs->argv[1]);
189 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
190 error(_("Didn't get START command"));
196 switch(cmdargs->cmd) {
217 if(a >= cmdargs->argc) {
218 error(_("error [chunker PORT-WRITE: not enough args: handle]"));
221 handle = newstralloc(handle, cmdargs->argv[a++]);
223 if(a >= cmdargs->argc) {
224 error(_("error [chunker PORT-WRITE: not enough args: filename]"));
227 filename = newstralloc(filename, cmdargs->argv[a++]);
229 if(a >= cmdargs->argc) {
230 error(_("error [chunker PORT-WRITE: not enough args: hostname]"));
233 hostname = newstralloc(hostname, cmdargs->argv[a++]);
235 if(a >= cmdargs->argc) {
236 error(_("error [chunker PORT-WRITE: not enough args: features]"));
239 am_release_feature_set(their_features);
240 their_features = am_string_to_feature(cmdargs->argv[a++]);
242 if(a >= cmdargs->argc) {
243 error(_("error [chunker PORT-WRITE: not enough args: diskname]"));
246 diskname = newstralloc(diskname, cmdargs->argv[a++]);
249 qdiskname = quote_string(diskname); /* qdiskname is a global */
251 if(a >= cmdargs->argc) {
252 error(_("error [chunker PORT-WRITE: not enough args: level]"));
255 level = atoi(cmdargs->argv[a++]);
257 if(a >= cmdargs->argc) {
258 error(_("error [chunker PORT-WRITE: not enough args: dumpdate]"));
261 dumpdate = newstralloc(dumpdate, cmdargs->argv[a++]);
263 if(a >= cmdargs->argc) {
264 error(_("error [chunker PORT-WRITE: not enough args: chunksize]"));
267 chunksize = OFF_T_ATOI(cmdargs->argv[a++]);
268 chunksize = am_floor(chunksize, (off_t)DISK_BLOCK_KB);
270 if(a >= cmdargs->argc) {
271 error(_("error [chunker PORT-WRITE: not enough args: progname]"));
274 progname = newstralloc(progname, cmdargs->argv[a++]);
276 if(a >= cmdargs->argc) {
277 error(_("error [chunker PORT-WRITE: not enough args: use]"));
280 use = am_floor(OFF_T_ATOI(cmdargs->argv[a++]), DISK_BLOCK_KB);
282 if(a >= cmdargs->argc) {
283 error(_("error [chunker PORT-WRITE: not enough args: options]"));
286 options = newstralloc(options, cmdargs->argv[a++]);
288 if(a != cmdargs->argc) {
289 error(_("error [chunker PORT-WRITE: too many args: %d != %d]"),
294 if((infd = startup_chunker(filename, use, chunksize, &db)) < 0) {
295 q = quote_string(vstrallocf(_("[chunker startup failed: %s]"), errstr));
296 putresult(TRYAGAIN, "%s %s\n", handle, q);
297 error("startup_chunker failed: %s", errstr);
299 command_in_transit = NULL;
300 if(infd >= 0 && do_chunk(infd, &db)) {
301 char kb_str[NUM_STR_SIZE];
302 char kps_str[NUM_STR_SIZE];
305 runtime = stopclock();
306 rt = g_timeval_to_double(runtime);
307 g_snprintf(kb_str, SIZEOF(kb_str), "%lld",
308 (long long)(dumpsize - (off_t)headersize));
309 g_snprintf(kps_str, SIZEOF(kps_str), "%3.1lf",
310 isnormal(rt) ? (double)dumpsize / rt : 0.0);
311 errstr = newvstrallocf(errstr, "sec %s kb %s kps %s",
312 walltime_str(runtime), kb_str, kps_str);
313 m = vstrallocf("[%s]", errstr);
316 if(command_in_transit != NULL) {
317 cmdargs = command_in_transit;
318 command_in_transit = NULL;
322 switch(cmdargs->cmd) {
324 putresult(DONE, "%s %lld %s\n", handle,
325 (long long)(dumpsize - (off_t)headersize), q);
326 log_add(L_SUCCESS, "%s %s %s %d [%s]",
327 hostname, qdiskname, chunker_timestamp, level, errstr);
333 if(dumpsize > (off_t)DISK_BLOCK_KB) {
334 putresult(PARTIAL, "%s %lld %s\n", handle,
335 (long long)(dumpsize - (off_t)headersize),
337 log_add(L_PARTIAL, "%s %s %s %d [%s]",
338 hostname, qdiskname, chunker_timestamp, level, errstr);
341 errstr = newvstrallocf(errstr,
342 _("dumper returned %s"), cmdstr[cmdargs->cmd]);
344 m = vstrallocf("[%s]",errstr);
347 putresult(FAILED, "%s %s\n", handle, q);
348 log_add(L_FAIL, "%s %s %s %d [%s]",
349 hostname, qdiskname, chunker_timestamp, level, errstr);
354 } else if(infd != -2) {
356 m = vstrallocf("[%s]", errstr);
361 putresult(FAILED, "%s %s\n", handle, q);
363 log_add(L_FAIL, "%s %s %s %d [%s]",
364 hostname, qdiskname, chunker_timestamp, level, errstr);
372 if(cmdargs->argc >= 1) {
373 q = quote_string(cmdargs->argv[0]);
375 q = stralloc(_("(no input?)"));
377 putresult(BAD_COMMAND, "%s\n", q);
382 /* } while(cmdargs->cmd != QUIT); */
384 log_add(L_INFO, "pid-done %ld", (long)getpid());
387 amfree(chunker_timestamp);
395 free_cmdargs(cmdargs);
396 if (command_in_transit)
397 free_cmdargs(command_in_transit);
398 am_release_feature_set(their_features);
399 their_features = NULL;
403 return (0); /* exit */
407 * Returns a file descriptor to the incoming port
408 * on success, or -1 on error.
418 char *tmp_filename, *pc;
422 struct addrinfo *res;
425 if ((result = resolve_hostname("localhost", 0, &res, NULL) != 0)) {
426 errstr = newvstrallocf(errstr, _("could not resolve localhost: %s"),
427 gai_strerror(result));
430 data_socket = stream_server(res->ai_family, &data_port, 0,
432 if (res) freeaddrinfo(res);
434 if(data_socket < 0) {
435 errstr = vstrallocf(_("error creating stream server: %s"), strerror(errno));
439 putresult(PORT, "%d\n", data_port);
441 infd = stream_accept(data_socket, CONNECT_TIMEOUT, 0, STREAM_BUFSIZE);
444 errstr = vstrallocf(_("error accepting stream: %s"), strerror(errno));
448 tmp_filename = vstralloc(filename, ".tmp", NULL);
449 pc = strrchr(tmp_filename, '/');
450 g_assert(pc != NULL);
452 mkholdingdir(tmp_filename);
454 if ((outfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600)) < 0) {
455 int save_errno = errno;
456 char *m = vstrallocf(_("holding file \"%s\": %s"),
460 errstr = quote_string(m);
462 amfree(tmp_filename);
464 if(save_errno == ENOSPC) {
465 putresult(NO_ROOM, "%s %lld\n",
466 handle, (long long)use);
472 amfree(tmp_filename);
473 databuf_init(db, outfd, filename, use, chunksize);
484 char header_buf[DISK_BLOCK_BYTES];
488 dumpsize = dumpbytes = filesize = (off_t)0;
490 memset(header_buf, 0, sizeof(header_buf));
493 * The first thing we should receive is the file header, which we
494 * need to save into "file", as well as write out. Later, the
495 * chunk code will rewrite it.
497 nread = full_read(infd, header_buf, SIZEOF(header_buf));
498 if (nread != sizeof(header_buf)) {
500 errstr = vstrallocf(_("cannot read header: %s"), strerror(errno));
502 errstr = vstrallocf(_("cannot read header: got %zd bytes instead of %zd"),
503 nread, sizeof(header_buf));
507 parse_file_header(header_buf, &file, (size_t)nread);
508 if(write_tapeheader(db->fd, &file)) {
509 int save_errno = errno;
510 char *m = vstrallocf(_("write_tapeheader file %s: %s"),
511 db->filename, strerror(errno));
512 errstr = quote_string(m);
514 if(save_errno == ENOSPC) {
515 putresult(NO_ROOM, "%s %lld\n", handle,
516 (long long)(db->use+db->split_size-dumpsize));
520 dumpsize += (off_t)DISK_BLOCK_KB;
521 filesize = (off_t)DISK_BLOCK_KB;
522 headersize += DISK_BLOCK_KB;
525 * We've written the file header. Now, just write data until the
528 while ((nread = full_read(infd, db->buf,
529 (size_t)(db->datalimit - db->datain))) > 0) {
531 while(db->dataout < db->datain) {
532 if(!databuf_flush(db)) {
537 while(db->dataout < db->datain) {
538 if(!databuf_flush(db)) {
542 if(dumpbytes > (off_t)0) {
543 dumpsize += (off_t)1; /* count partial final KByte */
544 filesize += (off_t)1;
550 * Initialize a databuf. Takes a writeable file descriptor.
561 db->filename = stralloc(filename);
562 db->filename_seq = (off_t)0;
563 db->chunk_size = chunk_size;
564 db->split_size = (db->chunk_size > use) ? use : db->chunk_size;
565 db->use = (use > db->split_size) ? use - db->split_size : (off_t)0;
566 db->datain = db->dataout = db->buf;
567 db->datalimit = db->buf + SIZEOF(db->buf);
572 * Write out the buffer to the backing file
578 struct cmdargs *cmdargs = NULL;
582 char *arg_filename = NULL;
583 char *new_filename = NULL;
584 char *tmp_filename = NULL;
585 char sequence[NUM_STR_SIZE];
587 filetype_t save_type;
593 * If there's no data, do nothing.
595 if (db->dataout >= db->datain) {
600 * See if we need to split this file.
602 while (db->split_size > (off_t)0 && dumpsize >= db->split_size) {
603 if( db->use == (off_t)0 ) {
605 * Probably no more space on this disk. Request some more.
607 putresult(RQ_MORE_DISK, "%s\n", handle);
609 if(command_in_transit == NULL &&
610 (cmdargs->cmd == DONE || cmdargs->cmd == TRYAGAIN || cmdargs->cmd == FAILED)) {
611 command_in_transit = cmdargs;
614 if(cmdargs->cmd == CONTINUE) {
622 a = 2; /* skip CONTINUE and serial */
624 if(a >= cmdargs->argc) {
625 error(_("error [chunker CONTINUE: not enough args: filename]"));
628 arg_filename = newstralloc(arg_filename, cmdargs->argv[a++]);
630 if(a >= cmdargs->argc) {
631 error(_("error [chunker CONTINUE: not enough args: chunksize]"));
634 db->chunk_size = OFF_T_ATOI(cmdargs->argv[a++]);
635 db->chunk_size = am_floor(db->chunk_size, (off_t)DISK_BLOCK_KB);
637 if(a >= cmdargs->argc) {
638 error(_("error [chunker CONTINUE: not enough args: use]"));
641 db->use = OFF_T_ATOI(cmdargs->argv[a++]);
643 if(a != cmdargs->argc) {
644 error(_("error [chunker CONTINUE: too many args: %d != %d]"),
649 if(strcmp(db->filename, arg_filename) == 0) {
651 * Same disk, so use what room is left up to the
652 * next chunk boundary or the amount we were given,
655 left_in_chunk = db->chunk_size - filesize;
656 if(left_in_chunk > db->use) {
657 db->split_size += db->use;
660 db->split_size += left_in_chunk;
661 db->use -= left_in_chunk;
663 if(left_in_chunk > (off_t)0) {
665 * We still have space in this chunk.
671 * Different disk, so use new file.
673 db->filename = newstralloc(db->filename, arg_filename);
675 } else if(cmdargs->cmd == ABORT) {
677 errstr = newstralloc(errstr, cmdargs->argv[1]);
678 putresult(ABORT_FINISHED, "%s\n", handle);
682 if(cmdargs->argc >= 1) {
683 q = quote_string(cmdargs->argv[0]);
685 q = stralloc(_("(no input?)"));
687 error(_("error [bad command after RQ-MORE-DISK: \"%s\"]"), q);
693 * Time to use another file.
697 * First, open the new chunk file, and give it a new header
698 * that has no cont_filename pointer.
700 g_snprintf(sequence, SIZEOF(sequence), "%d", db->filename_seq);
701 new_filename = newvstralloc(new_filename,
706 tmp_filename = newvstralloc(tmp_filename,
710 pc = strrchr(tmp_filename, '/');
711 g_assert(pc != NULL); /* Only a problem if db->filename has no /. */
713 mkholdingdir(tmp_filename);
715 newfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
717 int save_errno = errno;
720 if(save_errno == ENOSPC) {
721 putresult(NO_ROOM, "%s %lld\n", handle,
722 (long long)(db->use+db->split_size-dumpsize));
723 db->use = (off_t)0; /* force RQ_MORE_DISK */
724 db->split_size = dumpsize;
727 m = vstrallocf(_("creating chunk holding file \"%s\": %s"),
730 errstr = quote_string(m);
736 save_type = file.type;
737 file.type = F_CONT_DUMPFILE;
738 file.cont_filename[0] = '\0';
739 if(write_tapeheader(newfd, &file)) {
740 int save_errno = errno;
744 if(save_errno == ENOSPC) {
745 putresult(NO_ROOM, "%s %lld\n", handle,
746 (long long)(db->use+db->split_size-dumpsize));
747 db->use = (off_t)0; /* force RQ_MORE DISK */
748 db->split_size = dumpsize;
751 m = vstrallocf(_("write_tapeheader file %s: %s"),
754 errstr = quote_string(m);
761 * Now, update the header of the current file to point
762 * to the next chunk, and then close it.
764 if (lseek(db->fd, (off_t)0, SEEK_SET) < (off_t)0) {
765 char *m = vstrallocf(_("lseek holding file %s: %s"),
768 errstr = quote_string(m);
775 file.type = save_type;
776 strncpy(file.cont_filename, new_filename, SIZEOF(file.cont_filename));
777 file.cont_filename[SIZEOF(file.cont_filename)-1] = '\0';
778 if(write_tapeheader(db->fd, &file)) {
779 char * m = vstrallocf(_("write_tapeheader file \"%s\": %s"),
782 errstr = quote_string(m);
785 unlink(tmp_filename);
789 file.type = F_CONT_DUMPFILE;
792 * Now shift the file descriptor.
799 * Update when we need to chunk again
801 if(db->use <= (off_t)DISK_BLOCK_KB) {
803 * Cheat and use one more block than allowed so we can make
806 db->split_size += (off_t)(2 * DISK_BLOCK_KB);
808 } else if(db->chunk_size > db->use) {
809 db->split_size += db->use;
812 db->split_size += db->chunk_size;
813 db->use -= db->chunk_size;
817 amfree(tmp_filename);
818 amfree(new_filename);
819 dumpsize += (off_t)DISK_BLOCK_KB;
820 filesize = (off_t)DISK_BLOCK_KB;
821 headersize += DISK_BLOCK_KB;
826 * Write out the buffer
828 written = full_write(db->fd, db->dataout,
829 (size_t)(db->datain - db->dataout));
831 db->dataout += written;
832 dumpbytes += (off_t)written;
834 dumpsize += (dumpbytes / (off_t)1024);
835 filesize += (dumpbytes / (off_t)1024);
838 if (errno != ENOSPC) {
839 char *m = vstrallocf(_("data write: %s"), strerror(errno));
840 errstr = quote_string(m);
847 * NO-ROOM is informational only. Later, RQ_MORE_DISK will be
848 * issued to use another holding disk.
850 putresult(NO_ROOM, "%s %lld\n", handle,
851 (long long)(db->use+db->split_size-dumpsize));
852 db->use = (off_t)0; /* force RQ_MORE_DISK */
853 db->split_size = dumpsize;
856 if (db->datain == db->dataout) {
858 * We flushed the whole buffer so reset to use it all.
860 db->datain = db->dataout = db->buf;
866 free_cmdargs(cmdargs);
867 amfree(new_filename);
868 /*@i@*/ amfree(tmp_filename);
869 amfree(arg_filename);
875 * Send an Amanda dump header to the output file and set file->blocksize
885 file->blocksize = DISK_BLOCK_BYTES;
886 buffer = build_header(file, DISK_BLOCK_BYTES);
888 written = full_write(outfd, buffer, DISK_BLOCK_BYTES);
890 if(written == DISK_BLOCK_BYTES) return 0;
892 /* fake ENOSPC when we get a short write without errno set */