2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1999 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
26 /* $Id: chunker.c,v 1.36 2006/08/24 11:23:32 martinea Exp $
28 * requests remote amandad processes to dump filesystems
40 #include "fileheader.h"
41 #include "amfeatures.h"
42 #include "server_util.h"
45 #include "timestamp.h"
47 #define chunker_debug(i, ...) do { \
48 if ((i) <= debug_chunker) { \
49 dbprintf(__VA_ARGS__); \
61 #define CONNECT_TIMEOUT 5*60
63 #define STARTUP_TIMEOUT 60
66 int fd; /* file to flush to */
67 char *filename; /* name of what fd points to */
68 int filename_seq; /* for chunking */
69 off_t split_size; /* when to chunk */
70 off_t chunk_size; /* size of each chunk */
71 off_t use; /* size to use on this disk */
72 char buf[DISK_BLOCK_BYTES];
73 char *datain; /* data buffer markers */
78 static char *handle = NULL;
80 static char *errstr = NULL;
81 static int abort_pending;
82 static off_t dumpsize;
83 static unsigned long headersize;
84 static off_t dumpbytes;
85 static off_t filesize;
87 static char *hostname = NULL;
88 static char *diskname = NULL;
89 static char *qdiskname = NULL;
90 static char *options = NULL;
91 static char *progname = NULL;
93 static char *dumpdate = NULL;
94 static struct cmdargs *command_in_transit = NULL;
95 static char *chunker_timestamp = NULL;
97 static dumpfile_t file;
100 int main(int, char **);
101 static ssize_t write_tapeheader(int, dumpfile_t *);
102 static void databuf_init(struct databuf *, int, char *, off_t, off_t);
103 static int databuf_flush(struct databuf *);
105 static int startup_chunker(char *, off_t, off_t, struct databuf *, int *);
106 static int do_chunk(int, struct databuf *, int);
114 static struct databuf db;
115 struct cmdargs *cmdargs;
118 char *filename = NULL;
119 off_t chunksize, use;
121 am_feature_t *their_features = NULL;
123 config_overrides_t *cfg_ovr = NULL;
124 char *cfg_opt = NULL;
129 * Configure program for internationalization:
130 * 1) Only set the message locale for now.
131 * 2) Set textdomain for all amanda related programs to "amanda"
132 * We don't want to be forced to support dozens of message catalogs.
134 setlocale(LC_MESSAGES, "C");
135 textdomain("amanda");
139 set_pname("chunker");
141 dbopen(DBG_SUBDIR_SERVER);
143 /* Don't die when child closes pipe */
144 signal(SIGPIPE, SIG_IGN);
146 add_amanda_log_handler(amanda_log_stderr);
147 add_amanda_log_handler(amanda_log_trace_log);
149 cfg_ovr = extract_commandline_config_overrides(&argc, &argv);
154 set_config_overrides(cfg_ovr);
155 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
157 if (config_errors(NULL) >= CFGERR_WARNINGS) {
158 config_print_errors();
159 if (config_errors(NULL) >= CFGERR_ERRORS) {
160 g_critical(_("errors processing config file"));
164 safe_cd(); /* do this *after* config_init() */
166 check_running_as(RUNNING_AS_DUMPUSER);
168 dbrename(get_config_name(), DBG_SUBDIR_SERVER);
170 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
172 _("%s: pid %ld executable %s version %s\n"),
173 get_pname(), (long) getpid(),
177 /* now, make sure we are a valid user */
179 signal(SIGPIPE, SIG_IGN);
180 signal(SIGCHLD, SIG_IGN);
183 if(cmdargs->cmd == START) {
184 if(cmdargs->argc <= 1)
185 error(_("error [dumper START: not enough args: timestamp]"));
186 chunker_timestamp = newstralloc(chunker_timestamp, cmdargs->argv[1]);
189 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
190 error(_("Didn't get START command"));
196 switch(cmdargs->cmd) {
217 if(a >= cmdargs->argc) {
218 error(_("error [chunker PORT-WRITE: not enough args: handle]"));
221 handle = newstralloc(handle, cmdargs->argv[a++]);
223 if(a >= cmdargs->argc) {
224 error(_("error [chunker PORT-WRITE: not enough args: filename]"));
227 filename = newstralloc(filename, cmdargs->argv[a++]);
229 if(a >= cmdargs->argc) {
230 error(_("error [chunker PORT-WRITE: not enough args: hostname]"));
233 hostname = newstralloc(hostname, cmdargs->argv[a++]);
235 if(a >= cmdargs->argc) {
236 error(_("error [chunker PORT-WRITE: not enough args: features]"));
239 am_release_feature_set(their_features);
240 their_features = am_string_to_feature(cmdargs->argv[a++]);
241 if (!their_features) {
242 error(_("error [chunker PORT-WRITE: invalid feature string]"));
246 if(a >= cmdargs->argc) {
247 error(_("error [chunker PORT-WRITE: not enough args: diskname]"));
250 diskname = newstralloc(diskname, cmdargs->argv[a++]);
253 qdiskname = quote_string(diskname); /* qdiskname is a global */
255 if(a >= cmdargs->argc) {
256 error(_("error [chunker PORT-WRITE: not enough args: level]"));
259 level = atoi(cmdargs->argv[a++]);
261 if(a >= cmdargs->argc) {
262 error(_("error [chunker PORT-WRITE: not enough args: dumpdate]"));
265 dumpdate = newstralloc(dumpdate, cmdargs->argv[a++]);
267 if(a >= cmdargs->argc) {
268 error(_("error [chunker PORT-WRITE: not enough args: chunksize]"));
271 chunksize = OFF_T_ATOI(cmdargs->argv[a++]);
272 chunksize = am_floor(chunksize, (off_t)DISK_BLOCK_KB);
274 if(a >= cmdargs->argc) {
275 error(_("error [chunker PORT-WRITE: not enough args: progname]"));
278 progname = newstralloc(progname, cmdargs->argv[a++]);
280 if(a >= cmdargs->argc) {
281 error(_("error [chunker PORT-WRITE: not enough args: use]"));
284 use = am_floor(OFF_T_ATOI(cmdargs->argv[a++]), DISK_BLOCK_KB);
286 if(a >= cmdargs->argc) {
287 error(_("error [chunker PORT-WRITE: not enough args: options]"));
290 options = newstralloc(options, cmdargs->argv[a++]);
292 if(a != cmdargs->argc) {
293 error(_("error [chunker PORT-WRITE: too many args: %d != %d]"),
298 if ((header_fd = startup_chunker(filename, use, chunksize, &db,
299 &data_socket)) < 0) {
300 q = quote_string(vstrallocf(_("[chunker startup failed: %s]"), errstr));
301 putresult(TRYAGAIN, "%s %s\n", handle, q);
302 error("startup_chunker failed: %s", errstr);
304 command_in_transit = NULL;
305 if (header_fd >= 0 && do_chunk(header_fd, &db, data_socket)) {
306 char kb_str[NUM_STR_SIZE];
307 char kps_str[NUM_STR_SIZE];
310 runtime = stopclock();
311 rt = g_timeval_to_double(runtime);
312 g_snprintf(kb_str, SIZEOF(kb_str), "%lld",
313 (long long)(dumpsize - (off_t)headersize));
314 g_snprintf(kps_str, SIZEOF(kps_str), "%3.1lf",
315 isnormal(rt) ? (double)dumpsize / rt : 0.0);
316 errstr = newvstrallocf(errstr, "sec %s kb %s kps %s",
317 walltime_str(runtime), kb_str, kps_str);
318 m = vstrallocf("[%s]", errstr);
321 if(command_in_transit != NULL) {
322 cmdargs = command_in_transit;
323 command_in_transit = NULL;
327 switch(cmdargs->cmd) {
329 putresult(DONE, "%s %lld %s\n", handle,
330 (long long)(dumpsize - (off_t)headersize), q);
331 log_add(L_SUCCESS, "%s %s %s %d [%s]",
332 hostname, qdiskname, chunker_timestamp, level, errstr);
338 if(dumpsize > (off_t)DISK_BLOCK_KB) {
339 putresult(PARTIAL, "%s %lld %s\n", handle,
340 (long long)(dumpsize - (off_t)headersize),
342 log_add(L_PARTIAL, "%s %s %s %d [%s]",
343 hostname, qdiskname, chunker_timestamp, level, errstr);
346 errstr = newvstrallocf(errstr,
347 _("dumper returned %s"), cmdstr[cmdargs->cmd]);
349 m = vstrallocf("[%s]",errstr);
352 putresult(FAILED, "%s %s\n", handle, q);
353 log_add(L_FAIL, "%s %s %s %d [%s]",
354 hostname, qdiskname, chunker_timestamp, level, errstr);
359 } else if (header_fd != -2) {
361 m = vstrallocf("[%s]", errstr);
366 putresult(FAILED, "%s %s\n", handle, q);
368 log_add(L_FAIL, "%s %s %s %d [%s]",
369 hostname, qdiskname, chunker_timestamp, level, errstr);
377 if(cmdargs->argc >= 1) {
378 q = quote_string(cmdargs->argv[0]);
380 q = stralloc(_("(no input?)"));
382 putresult(BAD_COMMAND, "%s\n", q);
387 /* } while(cmdargs->cmd != QUIT); */
389 log_add(L_INFO, "pid-done %ld", (long)getpid());
392 amfree(chunker_timestamp);
400 free_cmdargs(cmdargs);
401 if (command_in_transit)
402 free_cmdargs(command_in_transit);
403 am_release_feature_set(their_features);
404 their_features = NULL;
408 return (0); /* exit */
412 * Returns a file descriptor to the incoming port
413 * on success, or -1 on error.
423 int header_fd, outfd;
424 char *tmp_filename, *pc;
425 in_port_t header_port, data_port;
426 int header_socket, data_socket;
428 struct addrinfo *res;
432 if ((result = resolve_hostname("localhost", 0, &res, NULL) != 0)) {
433 errstr = newvstrallocf(errstr, _("could not resolve localhost: %s"),
434 gai_strerror(result));
437 header_socket = stream_server(res->ai_family, &header_port, 0,
439 data_socket = stream_server(res->ai_family, &data_port, 0,
441 if (res) freeaddrinfo(res);
443 if (header_socket < 0) {
444 errstr = vstrallocf(_("error creating header stream server: %s"), strerror(errno));
449 if (data_socket < 0) {
450 errstr = vstrallocf(_("error creating data stream server: %s"), strerror(errno));
451 aclose(header_socket);
455 putresult(PORT, "%d 127.0.0.1:%d\n", header_port, data_port);
457 header_fd = stream_accept(header_socket, CONNECT_TIMEOUT, 0,
459 if (header_fd == -1) {
460 errstr = vstrallocf(_("error accepting header stream: %s"),
462 aclose(header_socket);
466 aclose(header_socket);
468 tmp_filename = vstralloc(filename, ".tmp", NULL);
469 pc = strrchr(tmp_filename, '/');
470 g_assert(pc != NULL);
472 mkholdingdir(tmp_filename);
474 if ((outfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600)) < 0) {
475 int save_errno = errno;
476 char *m = vstrallocf(_("holding file \"%s\": %s"),
480 errstr = quote_string(m);
482 amfree(tmp_filename);
485 if(save_errno == ENOSPC) {
486 putresult(NO_ROOM, "%s %lld\n",
487 handle, (long long)use);
493 amfree(tmp_filename);
494 databuf_init(db, outfd, filename, use, chunksize);
496 *datasocket = data_socket;
508 char header_buf[DISK_BLOCK_BYTES];
512 dumpsize = dumpbytes = filesize = (off_t)0;
514 memset(header_buf, 0, sizeof(header_buf));
517 * The first thing we should receive is the file header, which we
518 * need to save into "file", as well as write out. Later, the
519 * chunk code will rewrite it.
521 nread = full_read(header_fd, header_buf, SIZEOF(header_buf));
522 if (nread != sizeof(header_buf)) {
524 errstr = vstrallocf(_("cannot read header: %s"), strerror(errno));
526 errstr = vstrallocf(_("cannot read header: got %zd bytes instead of %zd"),
527 nread, sizeof(header_buf));
532 parse_file_header(header_buf, &file, (size_t)nread);
533 if(write_tapeheader(db->fd, &file)) {
534 int save_errno = errno;
535 char *m = vstrallocf(_("write_tapeheader file %s: %s"),
536 db->filename, strerror(errno));
537 errstr = quote_string(m);
539 if(save_errno == ENOSPC) {
540 putresult(NO_ROOM, "%s %lld\n", handle,
541 (long long)(db->use+db->split_size-dumpsize));
546 dumpsize += (off_t)DISK_BLOCK_KB;
547 filesize = (off_t)DISK_BLOCK_KB;
548 headersize += DISK_BLOCK_KB;
550 /* open the data socket */
551 data_fd = stream_accept(data_socket, CONNECT_TIMEOUT, 0, STREAM_BUFSIZE);
554 errstr = vstrallocf(_("error accepting data stream: %s"),
562 * We've written the file header. Now, just write data until the
565 while ((nread = full_read(data_fd, db->buf,
566 (size_t)(db->datalimit - db->datain))) > 0) {
568 while(db->dataout < db->datain) {
569 if(!databuf_flush(db)) {
574 while(db->dataout < db->datain) {
575 if(!databuf_flush(db)) {
579 if(dumpbytes > (off_t)0) {
580 dumpsize += (off_t)1; /* count partial final KByte */
581 filesize += (off_t)1;
587 * Initialize a databuf. Takes a writeable file descriptor.
598 db->filename = stralloc(filename);
599 db->filename_seq = (off_t)0;
600 db->chunk_size = chunk_size;
601 db->split_size = (db->chunk_size > use) ? use : db->chunk_size;
602 db->use = (use > db->split_size) ? use - db->split_size : (off_t)0;
603 db->datain = db->dataout = db->buf;
604 db->datalimit = db->buf + SIZEOF(db->buf);
609 * Write out the buffer to the backing file
615 struct cmdargs *cmdargs = NULL;
617 size_t size_to_write;
620 char *arg_filename = NULL;
621 char *new_filename = NULL;
622 char *tmp_filename = NULL;
623 char sequence[NUM_STR_SIZE];
625 filetype_t save_type;
631 * If there's no data, do nothing.
633 if (db->dataout >= db->datain) {
638 * See if we need to split this file.
640 while (db->split_size > (off_t)0 && dumpsize >= db->split_size) {
641 if( db->use == (off_t)0 ) {
643 * Probably no more space on this disk. Request some more.
645 putresult(RQ_MORE_DISK, "%s\n", handle);
647 if(command_in_transit == NULL &&
648 (cmdargs->cmd == DONE || cmdargs->cmd == TRYAGAIN || cmdargs->cmd == FAILED)) {
649 command_in_transit = cmdargs;
652 if(cmdargs->cmd == CONTINUE) {
660 a = 2; /* skip CONTINUE and serial */
662 if(a >= cmdargs->argc) {
663 error(_("error [chunker CONTINUE: not enough args: filename]"));
666 arg_filename = newstralloc(arg_filename, cmdargs->argv[a++]);
668 if(a >= cmdargs->argc) {
669 error(_("error [chunker CONTINUE: not enough args: chunksize]"));
672 db->chunk_size = OFF_T_ATOI(cmdargs->argv[a++]);
673 db->chunk_size = am_floor(db->chunk_size, (off_t)DISK_BLOCK_KB);
675 if(a >= cmdargs->argc) {
676 error(_("error [chunker CONTINUE: not enough args: use]"));
679 db->use = OFF_T_ATOI(cmdargs->argv[a++]);
681 if(a != cmdargs->argc) {
682 error(_("error [chunker CONTINUE: too many args: %d != %d]"),
687 if(strcmp(db->filename, arg_filename) == 0) {
689 * Same disk, so use what room is left up to the
690 * next chunk boundary or the amount we were given,
693 left_in_chunk = db->chunk_size - filesize;
694 if(left_in_chunk > db->use) {
695 db->split_size += db->use;
698 db->split_size += left_in_chunk;
699 db->use -= left_in_chunk;
701 if(left_in_chunk > (off_t)0) {
703 * We still have space in this chunk.
709 * Different disk, so use new file.
711 db->filename = newstralloc(db->filename, arg_filename);
713 } else if(cmdargs->cmd == ABORT) {
715 errstr = newstralloc(errstr, cmdargs->argv[1]);
716 putresult(ABORT_FINISHED, "%s\n", handle);
720 if(cmdargs->argc >= 1) {
721 q = quote_string(cmdargs->argv[0]);
723 q = stralloc(_("(no input?)"));
725 error(_("error [bad command after RQ-MORE-DISK: \"%s\"]"), q);
731 * Time to use another file.
735 * First, open the new chunk file, and give it a new header
736 * that has no cont_filename pointer.
738 g_snprintf(sequence, SIZEOF(sequence), "%d", db->filename_seq);
739 new_filename = newvstralloc(new_filename,
744 tmp_filename = newvstralloc(tmp_filename,
748 pc = strrchr(tmp_filename, '/');
749 g_assert(pc != NULL); /* Only a problem if db->filename has no /. */
751 mkholdingdir(tmp_filename);
753 newfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
755 int save_errno = errno;
758 if(save_errno == ENOSPC) {
759 putresult(NO_ROOM, "%s %lld\n", handle,
760 (long long)(db->use+db->split_size-dumpsize));
761 db->use = (off_t)0; /* force RQ_MORE_DISK */
762 db->split_size = dumpsize;
765 m = vstrallocf(_("creating chunk holding file \"%s\": %s"),
768 errstr = quote_string(m);
774 save_type = file.type;
775 file.type = F_CONT_DUMPFILE;
776 file.cont_filename[0] = '\0';
777 if(write_tapeheader(newfd, &file)) {
778 int save_errno = errno;
782 if(save_errno == ENOSPC) {
783 putresult(NO_ROOM, "%s %lld\n", handle,
784 (long long)(db->use+db->split_size-dumpsize));
785 db->use = (off_t)0; /* force RQ_MORE DISK */
786 db->split_size = dumpsize;
789 m = vstrallocf(_("write_tapeheader file %s: %s"),
792 errstr = quote_string(m);
799 * Now, update the header of the current file to point
800 * to the next chunk, and then close it.
802 if (lseek(db->fd, (off_t)0, SEEK_SET) < (off_t)0) {
803 char *m = vstrallocf(_("lseek holding file %s: %s"),
806 errstr = quote_string(m);
813 file.type = save_type;
814 strncpy(file.cont_filename, new_filename, SIZEOF(file.cont_filename));
815 file.cont_filename[SIZEOF(file.cont_filename)-1] = '\0';
816 if(write_tapeheader(db->fd, &file)) {
817 char * m = vstrallocf(_("write_tapeheader file \"%s\": %s"),
820 errstr = quote_string(m);
823 unlink(tmp_filename);
827 file.type = F_CONT_DUMPFILE;
830 * Now shift the file descriptor.
837 * Update when we need to chunk again
839 if(db->use <= (off_t)DISK_BLOCK_KB) {
841 * Cheat and use one more block than allowed so we can make
844 db->split_size += (off_t)(2 * DISK_BLOCK_KB);
846 } else if(db->chunk_size > db->use) {
847 db->split_size += db->use;
850 db->split_size += db->chunk_size;
851 db->use -= db->chunk_size;
855 amfree(tmp_filename);
856 amfree(new_filename);
857 dumpsize += (off_t)DISK_BLOCK_KB;
858 filesize = (off_t)DISK_BLOCK_KB;
859 headersize += DISK_BLOCK_KB;
864 * Write out the buffer
866 size_to_write = (size_t)(db->datain - db->dataout);
867 written = full_write(db->fd, db->dataout, size_to_write);
869 db->dataout += written;
870 dumpbytes += (off_t)written;
872 dumpsize += (dumpbytes / (off_t)1024);
873 filesize += (dumpbytes / (off_t)1024);
875 if (written < size_to_write) {
876 if (errno != ENOSPC) {
877 char *m = vstrallocf(_("data write: %s"), strerror(errno));
878 errstr = quote_string(m);
885 * NO-ROOM is informational only. Later, RQ_MORE_DISK will be
886 * issued to use another holding disk.
888 putresult(NO_ROOM, "%s %lld\n", handle,
889 (long long)(db->use+db->split_size-dumpsize));
890 db->use = (off_t)0; /* force RQ_MORE_DISK */
891 db->split_size = dumpsize;
894 if (db->datain == db->dataout) {
896 * We flushed the whole buffer so reset to use it all.
898 db->datain = db->dataout = db->buf;
904 free_cmdargs(cmdargs);
905 amfree(new_filename);
906 /*@i@*/ amfree(tmp_filename);
907 amfree(arg_filename);
913 * Send an Amanda dump header to the output file and set file->blocksize
923 file->blocksize = DISK_BLOCK_BYTES;
924 if (debug_chunker > 1)
925 dump_dumpfile_t(file);
926 buffer = build_header(file, NULL, DISK_BLOCK_BYTES);
927 if (!buffer) /* this shouldn't happen */
928 error(_("header does not fit in %zd bytes"), (size_t)DISK_BLOCK_BYTES);
930 written = full_write(outfd, buffer, DISK_BLOCK_BYTES);
932 if(written == DISK_BLOCK_BYTES) return 0;
934 /* fake ENOSPC when we get a short write without errno set */