2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1999 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
26 /* $Id: chunker.c,v 1.36 2006/08/24 11:23:32 martinea Exp $
28 * requests remote amandad processes to dump filesystems
42 #include "fileheader.h"
43 #include "amfeatures.h"
44 #include "server_util.h"
47 #include "timestamp.h"
49 #define chunker_debug(i, ...) do { \
50 if ((i) <= debug_chunker) { \
51 dbprintf(__VA_ARGS__); \
63 #define CONNECT_TIMEOUT 5*60
65 #define STARTUP_TIMEOUT 60
68 int fd; /* file to flush to */
69 char *filename; /* name of what fd points to */
70 int filename_seq; /* for chunking */
71 off_t split_size; /* when to chunk */
72 off_t chunk_size; /* size of each chunk */
73 off_t use; /* size to use on this disk */
74 char buf[DISK_BLOCK_BYTES];
75 char *datain; /* data buffer markers */
80 static char *handle = NULL;
82 static char *errstr = NULL;
83 static int abort_pending;
84 static off_t dumpsize;
85 static unsigned long headersize;
86 static off_t dumpbytes;
87 static off_t filesize;
89 static char *hostname = NULL;
90 static char *diskname = NULL;
91 static char *qdiskname = NULL;
92 static char *options = NULL;
93 static char *progname = NULL;
95 static char *dumpdate = NULL;
96 static int command_in_transit;
97 static char *chunker_timestamp = NULL;
99 static dumpfile_t file;
101 /* local functions */
102 int main(int, char **);
103 static ssize_t write_tapeheader(int, dumpfile_t *);
104 static void databuf_init(struct databuf *, int, char *, off_t, off_t);
105 static int databuf_flush(struct databuf *);
107 static int startup_chunker(char *, off_t, off_t, struct databuf *);
108 static int do_chunk(int, struct databuf *);
116 static struct databuf db;
117 struct cmdargs cmdargs;
121 char *filename = NULL;
122 char *qfilename = NULL;
123 off_t chunksize, use;
125 am_feature_t *their_features = NULL;
127 config_overwrites_t *cfg_ovr = NULL;
128 char *cfg_opt = NULL;
131 * Configure program for internationalization:
132 * 1) Only set the message locale for now.
133 * 2) Set textdomain for all amanda related programs to "amanda"
134 * We don't want to be forced to support dozens of message catalogs.
136 setlocale(LC_MESSAGES, "C");
137 textdomain("amanda");
141 set_pname("chunker");
143 dbopen(DBG_SUBDIR_SERVER);
145 /* Don't die when child closes pipe */
146 signal(SIGPIPE, SIG_IGN);
148 erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE);
149 set_logerror(logerror);
151 cfg_ovr = extract_commandline_config_overwrites(&argc, &argv);
156 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD | CONFIG_INIT_FATAL,
158 apply_config_overwrites(cfg_ovr);
160 safe_cd(); /* do this *after* config_init() */
162 check_running_as(RUNNING_AS_DUMPUSER);
164 dbrename(config_name, DBG_SUBDIR_SERVER);
167 _("%s: pid %ld executable %s version %s\n"),
168 get_pname(), (long) getpid(),
172 /* now, make sure we are a valid user */
174 signal(SIGPIPE, SIG_IGN);
175 signal(SIGCHLD, SIG_IGN);
177 cmd = getcmd(&cmdargs);
179 if(cmdargs.argc <= 1)
180 error(_("error [dumper START: not enough args: timestamp]"));
181 chunker_timestamp = newstralloc(chunker_timestamp, cmdargs.argv[2]);
184 error(_("Didn't get START command"));
188 cmd = getcmd(&cmdargs);
209 cmdargs.argc++; /* true count of args */
212 if(a >= cmdargs.argc) {
213 error(_("error [chunker PORT-WRITE: not enough args: handle]"));
216 handle = newstralloc(handle, cmdargs.argv[a++]);
218 if(a >= cmdargs.argc) {
219 error(_("error [chunker PORT-WRITE: not enough args: filename]"));
222 qfilename = newstralloc(qfilename, cmdargs.argv[a++]);
223 if (filename != NULL)
225 filename = unquote_string(qfilename);
228 if(a >= cmdargs.argc) {
229 error(_("error [chunker PORT-WRITE: not enough args: hostname]"));
232 hostname = newstralloc(hostname, cmdargs.argv[a++]);
234 if(a >= cmdargs.argc) {
235 error(_("error [chunker PORT-WRITE: not enough args: features]"));
238 am_release_feature_set(their_features);
239 their_features = am_string_to_feature(cmdargs.argv[a++]);
241 if(a >= cmdargs.argc) {
242 error(_("error [chunker PORT-WRITE: not enough args: diskname]"));
245 qdiskname = newstralloc(qdiskname, cmdargs.argv[a++]);
246 if (diskname != NULL)
248 diskname = unquote_string(qdiskname);
250 if(a >= cmdargs.argc) {
251 error(_("error [chunker PORT-WRITE: not enough args: level]"));
254 level = atoi(cmdargs.argv[a++]);
256 if(a >= cmdargs.argc) {
257 error(_("error [chunker PORT-WRITE: not enough args: dumpdate]"));
260 dumpdate = newstralloc(dumpdate, cmdargs.argv[a++]);
262 if(a >= cmdargs.argc) {
263 error(_("error [chunker PORT-WRITE: not enough args: chunksize]"));
266 chunksize = OFF_T_ATOI(cmdargs.argv[a++]);
267 chunksize = am_floor(chunksize, (off_t)DISK_BLOCK_KB);
269 if(a >= cmdargs.argc) {
270 error(_("error [chunker PORT-WRITE: not enough args: progname]"));
273 progname = newstralloc(progname, cmdargs.argv[a++]);
275 if(a >= cmdargs.argc) {
276 error(_("error [chunker PORT-WRITE: not enough args: use]"));
279 use = am_floor(OFF_T_ATOI(cmdargs.argv[a++]), DISK_BLOCK_KB);
281 if(a >= cmdargs.argc) {
282 error(_("error [chunker PORT-WRITE: not enough args: options]"));
285 options = newstralloc(options, cmdargs.argv[a++]);
287 if(a != cmdargs.argc) {
288 error(_("error [chunker PORT-WRITE: too many args: %d != %d]"),
293 if((infd = startup_chunker(filename, use, chunksize, &db)) < 0) {
294 q = squotef(_("[chunker startup failed: %s]"), errstr);
295 putresult(TRYAGAIN, "%s %s\n", handle, q);
296 error("startup_chunker failed");
298 command_in_transit = -1;
299 if(infd >= 0 && do_chunk(infd, &db)) {
300 char kb_str[NUM_STR_SIZE];
301 char kps_str[NUM_STR_SIZE];
304 runtime = stopclock();
305 rt = g_timeval_to_double(runtime);
306 g_snprintf(kb_str, SIZEOF(kb_str), "%lld",
307 (long long)(dumpsize - (off_t)headersize));
308 g_snprintf(kps_str, SIZEOF(kps_str), "%3.1lf",
309 isnormal(rt) ? (double)dumpsize / rt : 0.0);
310 errstr = newvstrallocf(errstr, "sec %s kb %s kps %s",
311 walltime_str(runtime), kb_str, kps_str);
312 q = squotef("[%s]", errstr);
313 if(command_in_transit != -1)
314 cmd = command_in_transit;
316 cmd = getcmd(&cmdargs);
319 putresult(DONE, "%s %lld %s\n", handle,
320 (long long)(dumpsize - (off_t)headersize), q);
321 log_add(L_SUCCESS, "%s %s %s %d [%s]",
322 hostname, qdiskname, chunker_timestamp, level, errstr);
328 if(dumpsize > (off_t)DISK_BLOCK_KB) {
329 putresult(PARTIAL, "%s %lld %s\n", handle,
330 (long long)(dumpsize - (off_t)headersize),
332 log_add(L_PARTIAL, "%s %s %s %d [%s]",
333 hostname, qdiskname, chunker_timestamp, level, errstr);
336 errstr = newvstrallocf(errstr,
337 _("dumper returned %s"), cmdstr[cmd]);
339 q = squotef("[%s]",errstr);
340 putresult(FAILED, "%s %s\n", handle, q);
341 log_add(L_FAIL, "%s %s %s %d [%s]",
342 hostname, qdiskname, chunker_timestamp, level, errstr);
347 } else if(infd != -2) {
350 q = squotef("[%s]", errstr);
352 putresult(FAILED, "%s %s\n", handle, q);
353 log_add(L_FAIL, "%s %s %s %d [%s]",
354 hostname, qdiskname, chunker_timestamp, level, errstr);
363 if(cmdargs.argc >= 1) {
364 q = squote(cmdargs.argv[1]);
365 } else if(cmdargs.argc >= 0) {
366 q = squote(cmdargs.argv[0]);
368 q = stralloc(_("(no input?)"));
370 putresult(BAD_COMMAND, "%s\n", q);
375 /* } while(cmd != QUIT); */
378 amfree(chunker_timestamp);
386 am_release_feature_set(their_features);
387 their_features = NULL;
391 return (0); /* exit */
395 * Returns a file descriptor to the incoming port
396 * on success, or -1 on error.
406 char *tmp_filename, *pc;
410 struct addrinfo *res;
413 if ((result = resolve_hostname("localhost", 0, &res, NULL) != 0)) {
414 errstr = newvstrallocf(errstr, _("could not resolve localhost: %s"),
415 gai_strerror(result));
418 data_socket = stream_server(res->ai_family, &data_port, 0,
421 if(data_socket < 0) {
422 errstr = vstrallocf(_("error creating stream server: %s"), strerror(errno));
426 putresult(PORT, "%d\n", data_port);
428 infd = stream_accept(data_socket, CONNECT_TIMEOUT, 0, STREAM_BUFSIZE);
430 errstr = vstrallocf(_("error accepting stream: %s"), strerror(errno));
434 tmp_filename = vstralloc(filename, ".tmp", NULL);
435 pc = strrchr(tmp_filename, '/');
437 mkholdingdir(tmp_filename);
439 if ((outfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600)) < 0) {
440 int save_errno = errno;
442 errstr = squotef(_("holding file \"%s\": %s"),
445 amfree(tmp_filename);
447 if(save_errno == ENOSPC) {
448 putresult(NO_ROOM, "%s %lld\n",
449 handle, (long long)use);
455 amfree(tmp_filename);
456 databuf_init(db, outfd, filename, use, chunksize);
467 char header_buf[DISK_BLOCK_BYTES];
471 dumpsize = dumpbytes = filesize = (off_t)0;
473 memset(header_buf, 0, sizeof(header_buf));
476 * The first thing we should receive is the file header, which we
477 * need to save into "file", as well as write out. Later, the
478 * chunk code will rewrite it.
480 nread = fullread(infd, header_buf, SIZEOF(header_buf));
481 if (nread != DISK_BLOCK_BYTES) {
483 errstr = vstrallocf(_("cannot read header: %s"), strerror(errno));
485 errstr = vstrallocf(_("cannot read header: got %zd bytes instead of %d"),
491 parse_file_header(header_buf, &file, (size_t)nread);
492 if(write_tapeheader(db->fd, &file)) {
493 int save_errno = errno;
495 errstr = squotef(_("write_tapeheader file %s: %s"),
496 db->filename, strerror(errno));
497 if(save_errno == ENOSPC) {
498 putresult(NO_ROOM, "%s %lld\n", handle,
499 (long long)(db->use+db->split_size-dumpsize));
503 dumpsize += (off_t)DISK_BLOCK_KB;
504 filesize = (off_t)DISK_BLOCK_KB;
505 headersize += DISK_BLOCK_KB;
508 * We've written the file header. Now, just write data until the
511 while ((nread = fullread(infd, db->buf,
512 (size_t)(db->datalimit - db->datain))) > 0) {
514 while(db->dataout < db->datain) {
515 if(!databuf_flush(db)) {
520 while(db->dataout < db->datain) {
521 if(!databuf_flush(db)) {
525 if(dumpbytes > (off_t)0) {
526 dumpsize += (off_t)1; /* count partial final KByte */
527 filesize += (off_t)1;
533 * Initialize a databuf. Takes a writeable file descriptor.
544 db->filename = stralloc(filename);
545 db->filename_seq = (off_t)0;
546 db->chunk_size = chunk_size;
547 db->split_size = (db->chunk_size > use) ? use : db->chunk_size;
548 db->use = (use > db->split_size) ? use - db->split_size : (off_t)0;
549 db->datain = db->dataout = db->buf;
550 db->datalimit = db->buf + SIZEOF(db->buf);
555 * Write out the buffer to the backing file
561 struct cmdargs cmdargs;
565 char *arg_filename = NULL;
566 char *qarg_filename = NULL;
567 char *new_filename = NULL;
568 char *tmp_filename = NULL;
569 char sequence[NUM_STR_SIZE];
571 filetype_t save_type;
577 * If there's no data, do nothing.
579 if (db->dataout >= db->datain) {
584 * See if we need to split this file.
586 while (db->split_size > (off_t)0 && dumpsize >= db->split_size) {
587 if( db->use == (off_t)0 ) {
589 * Probably no more space on this disk. Request some more.
593 putresult(RQ_MORE_DISK, "%s\n", handle);
594 cmd = getcmd(&cmdargs);
595 if(command_in_transit == -1 &&
596 (cmd == DONE || cmd == TRYAGAIN || cmd == FAILED)) {
597 command_in_transit = cmd;
598 cmd = getcmd(&cmdargs);
600 if(cmd == CONTINUE) {
608 cmdargs.argc++; /* true count of args */
611 if(a >= cmdargs.argc) {
612 error(_("error [chunker CONTINUE: not enough args: filename]"));
615 qarg_filename = newstralloc(qarg_filename, cmdargs.argv[a++]);
616 if (arg_filename != NULL)
617 amfree(arg_filename);
618 arg_filename = unquote_string(qarg_filename);
620 if(a >= cmdargs.argc) {
621 error(_("error [chunker CONTINUE: not enough args: chunksize]"));
624 db->chunk_size = OFF_T_ATOI(cmdargs.argv[a++]);
625 db->chunk_size = am_floor(db->chunk_size, (off_t)DISK_BLOCK_KB);
627 if(a >= cmdargs.argc) {
628 error(_("error [chunker CONTINUE: not enough args: use]"));
631 db->use = OFF_T_ATOI(cmdargs.argv[a++]);
633 if(a != cmdargs.argc) {
634 error(_("error [chunker CONTINUE: too many args: %d != %d]"),
639 if(strcmp(db->filename, arg_filename) == 0) {
641 * Same disk, so use what room is left up to the
642 * next chunk boundary or the amount we were given,
645 left_in_chunk = db->chunk_size - filesize;
646 if(left_in_chunk > db->use) {
647 db->split_size += db->use;
650 db->split_size += left_in_chunk;
651 db->use -= left_in_chunk;
653 if(left_in_chunk > (off_t)0) {
655 * We still have space in this chunk.
661 * Different disk, so use new file.
663 db->filename = newstralloc(db->filename, arg_filename);
665 } else if(cmd == ABORT) {
667 errstr = newstralloc(errstr, "ERROR");
668 putresult(ABORT_FINISHED, "%s\n", handle);
672 if(cmdargs.argc >= 1) {
673 q = squote(cmdargs.argv[1]);
674 } else if(cmdargs.argc >= 0) {
675 q = squote(cmdargs.argv[0]);
677 q = stralloc(_("(no input?)"));
679 error(_("error [bad command after RQ-MORE-DISK: \"%s\"]"), q);
685 * Time to use another file.
689 * First, open the new chunk file, and give it a new header
690 * that has no cont_filename pointer.
692 g_snprintf(sequence, SIZEOF(sequence), "%d", db->filename_seq);
693 new_filename = newvstralloc(new_filename,
698 tmp_filename = newvstralloc(tmp_filename,
702 pc = strrchr(tmp_filename, '/');
704 mkholdingdir(tmp_filename);
706 newfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
708 int save_errno = errno;
710 if(save_errno == ENOSPC) {
711 putresult(NO_ROOM, "%s %lld\n", handle,
712 (long long)(db->use+db->split_size-dumpsize));
713 db->use = (off_t)0; /* force RQ_MORE_DISK */
714 db->split_size = dumpsize;
717 errstr = squotef(_("creating chunk holding file \"%s\": %s"),
724 save_type = file.type;
725 file.type = F_CONT_DUMPFILE;
726 file.cont_filename[0] = '\0';
727 if(write_tapeheader(newfd, &file)) {
728 int save_errno = errno;
731 if(save_errno == ENOSPC) {
732 putresult(NO_ROOM, "%s %lld\n", handle,
733 (long long)(db->use+db->split_size-dumpsize));
734 db->use = (off_t)0; /* force RQ_MORE DISK */
735 db->split_size = dumpsize;
738 errstr = squotef(_("write_tapeheader file %s: %s"),
746 * Now, update the header of the current file to point
747 * to the next chunk, and then close it.
749 if (lseek(db->fd, (off_t)0, SEEK_SET) < (off_t)0) {
750 errstr = squotef(_("lseek holding file %s: %s"),
758 file.type = save_type;
759 strncpy(file.cont_filename, new_filename, SIZEOF(file.cont_filename));
760 file.cont_filename[SIZEOF(file.cont_filename)] = '\0';
761 if(write_tapeheader(db->fd, &file)) {
762 errstr = squotef(_("write_tapeheader file \"%s\": %s"),
766 unlink(tmp_filename);
770 file.type = F_CONT_DUMPFILE;
773 * Now shift the file descriptor.
780 * Update when we need to chunk again
782 if(db->use <= (off_t)DISK_BLOCK_KB) {
784 * Cheat and use one more block than allowed so we can make
787 db->split_size += (off_t)(2 * DISK_BLOCK_KB);
789 } else if(db->chunk_size > db->use) {
790 db->split_size += db->use;
793 db->split_size += db->chunk_size;
794 db->use -= db->chunk_size;
798 amfree(tmp_filename);
799 amfree(new_filename);
800 dumpsize += (off_t)DISK_BLOCK_KB;
801 filesize = (off_t)DISK_BLOCK_KB;
802 headersize += DISK_BLOCK_KB;
807 * Write out the buffer
809 written = fullwrite(db->fd, db->dataout,
810 (size_t)(db->datain - db->dataout));
812 db->dataout += written;
813 dumpbytes += (off_t)written;
815 dumpsize += (dumpbytes / (off_t)1024);
816 filesize += (dumpbytes / (off_t)1024);
819 if (errno != ENOSPC) {
820 errstr = squotef(_("data write: %s"), strerror(errno));
826 * NO-ROOM is informational only. Later, RQ_MORE_DISK will be
827 * issued to use another holding disk.
829 putresult(NO_ROOM, "%s %lld\n", handle,
830 (long long)(db->use+db->split_size-dumpsize));
831 db->use = (off_t)0; /* force RQ_MORE_DISK */
832 db->split_size = dumpsize;
835 if (db->datain == db->dataout) {
837 * We flushed the whole buffer so reset to use it all.
839 db->datain = db->dataout = db->buf;
844 amfree(new_filename);
845 /*@i@*/ amfree(tmp_filename);
846 amfree(arg_filename);
847 amfree(qarg_filename);
853 * Send an Amanda dump header to the output file and set file->blocksize
863 file->blocksize = DISK_BLOCK_BYTES;
864 buffer = build_header(file, DISK_BLOCK_BYTES);
866 written = fullwrite(outfd, buffer, DISK_BLOCK_BYTES);
868 if(written == DISK_BLOCK_BYTES) return 0;
869 if(written < 0) return written;