2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1999 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
26 /* $Id: chunker.c,v 1.36 2006/08/24 11:23:32 martinea Exp $
28 * requests remote amandad processes to dump filesystems
41 #include "fileheader.h"
42 #include "amfeatures.h"
43 #include "server_util.h"
46 #include "timestamp.h"
48 #define chunker_debug(i, ...) do { \
49 if ((i) <= debug_chunker) { \
50 dbprintf(__VA_ARGS__); \
62 #define CONNECT_TIMEOUT 5*60
64 #define STARTUP_TIMEOUT 60
67 int fd; /* file to flush to */
68 char *filename; /* name of what fd points to */
69 int filename_seq; /* for chunking */
70 off_t split_size; /* when to chunk */
71 off_t chunk_size; /* size of each chunk */
72 off_t use; /* size to use on this disk */
73 char buf[DISK_BLOCK_BYTES];
74 char *datain; /* data buffer markers */
79 static char *handle = NULL;
81 static char *errstr = NULL;
82 static int abort_pending;
83 static off_t dumpsize;
84 static unsigned long headersize;
85 static off_t dumpbytes;
86 static off_t filesize;
88 static char *hostname = NULL;
89 static char *diskname = NULL;
90 static char *qdiskname = NULL;
91 static char *options = NULL;
92 static char *progname = NULL;
94 static char *dumpdate = NULL;
95 static struct cmdargs *command_in_transit = NULL;
96 static char *chunker_timestamp = NULL;
98 static dumpfile_t file;
100 /* local functions */
101 int main(int, char **);
102 static ssize_t write_tapeheader(int, dumpfile_t *);
103 static void databuf_init(struct databuf *, int, char *, off_t, off_t);
104 static int databuf_flush(struct databuf *);
106 static int startup_chunker(char *, off_t, off_t, struct databuf *);
107 static int do_chunk(int, struct databuf *);
115 static struct databuf db;
116 struct cmdargs *cmdargs;
119 char *filename = NULL;
120 off_t chunksize, use;
122 am_feature_t *their_features = NULL;
124 config_overwrites_t *cfg_ovr = NULL;
125 char *cfg_opt = NULL;
129 * Configure program for internationalization:
130 * 1) Only set the message locale for now.
131 * 2) Set textdomain for all amanda related programs to "amanda"
132 * We don't want to be forced to support dozens of message catalogs.
134 setlocale(LC_MESSAGES, "C");
135 textdomain("amanda");
139 set_pname("chunker");
141 dbopen(DBG_SUBDIR_SERVER);
143 /* Don't die when child closes pipe */
144 signal(SIGPIPE, SIG_IGN);
146 erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE);
147 set_logerror(logerror);
149 cfg_ovr = extract_commandline_config_overwrites(&argc, &argv);
154 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
155 apply_config_overwrites(cfg_ovr);
157 if (config_errors(NULL) >= CFGERR_WARNINGS) {
158 config_print_errors();
159 if (config_errors(NULL) >= CFGERR_ERRORS) {
160 g_critical(_("errors processing config file"));
164 safe_cd(); /* do this *after* config_init() */
166 check_running_as(RUNNING_AS_DUMPUSER);
168 dbrename(get_config_name(), DBG_SUBDIR_SERVER);
170 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
172 _("%s: pid %ld executable %s version %s\n"),
173 get_pname(), (long) getpid(),
177 /* now, make sure we are a valid user */
179 signal(SIGPIPE, SIG_IGN);
180 signal(SIGCHLD, SIG_IGN);
183 if(cmdargs->cmd == START) {
184 if(cmdargs->argc <= 1)
185 error(_("error [dumper START: not enough args: timestamp]"));
186 chunker_timestamp = newstralloc(chunker_timestamp, cmdargs->argv[1]);
189 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
190 error(_("Didn't get START command"));
196 switch(cmdargs->cmd) {
217 if(a >= cmdargs->argc) {
218 error(_("error [chunker PORT-WRITE: not enough args: handle]"));
221 handle = newstralloc(handle, cmdargs->argv[a++]);
223 if(a >= cmdargs->argc) {
224 error(_("error [chunker PORT-WRITE: not enough args: filename]"));
227 filename = newstralloc(filename, cmdargs->argv[a++]);
229 if(a >= cmdargs->argc) {
230 error(_("error [chunker PORT-WRITE: not enough args: hostname]"));
233 hostname = newstralloc(hostname, cmdargs->argv[a++]);
235 if(a >= cmdargs->argc) {
236 error(_("error [chunker PORT-WRITE: not enough args: features]"));
239 am_release_feature_set(their_features);
240 their_features = am_string_to_feature(cmdargs->argv[a++]);
241 if (!their_features) {
242 error(_("error [chunker PORT-WRITE: invalid feature string]"));
246 if(a >= cmdargs->argc) {
247 error(_("error [chunker PORT-WRITE: not enough args: diskname]"));
250 diskname = newstralloc(diskname, cmdargs->argv[a++]);
253 qdiskname = quote_string(diskname); /* qdiskname is a global */
255 if(a >= cmdargs->argc) {
256 error(_("error [chunker PORT-WRITE: not enough args: level]"));
259 level = atoi(cmdargs->argv[a++]);
261 if(a >= cmdargs->argc) {
262 error(_("error [chunker PORT-WRITE: not enough args: dumpdate]"));
265 dumpdate = newstralloc(dumpdate, cmdargs->argv[a++]);
267 if(a >= cmdargs->argc) {
268 error(_("error [chunker PORT-WRITE: not enough args: chunksize]"));
271 chunksize = OFF_T_ATOI(cmdargs->argv[a++]);
272 chunksize = am_floor(chunksize, (off_t)DISK_BLOCK_KB);
274 if(a >= cmdargs->argc) {
275 error(_("error [chunker PORT-WRITE: not enough args: progname]"));
278 progname = newstralloc(progname, cmdargs->argv[a++]);
280 if(a >= cmdargs->argc) {
281 error(_("error [chunker PORT-WRITE: not enough args: use]"));
284 use = am_floor(OFF_T_ATOI(cmdargs->argv[a++]), DISK_BLOCK_KB);
286 if(a >= cmdargs->argc) {
287 error(_("error [chunker PORT-WRITE: not enough args: options]"));
290 options = newstralloc(options, cmdargs->argv[a++]);
292 if(a != cmdargs->argc) {
293 error(_("error [chunker PORT-WRITE: too many args: %d != %d]"),
298 if((infd = startup_chunker(filename, use, chunksize, &db)) < 0) {
299 q = quote_string(vstrallocf(_("[chunker startup failed: %s]"), errstr));
300 putresult(TRYAGAIN, "%s %s\n", handle, q);
301 error("startup_chunker failed: %s", errstr);
303 command_in_transit = NULL;
304 if(infd >= 0 && do_chunk(infd, &db)) {
305 char kb_str[NUM_STR_SIZE];
306 char kps_str[NUM_STR_SIZE];
309 runtime = stopclock();
310 rt = g_timeval_to_double(runtime);
311 g_snprintf(kb_str, SIZEOF(kb_str), "%lld",
312 (long long)(dumpsize - (off_t)headersize));
313 g_snprintf(kps_str, SIZEOF(kps_str), "%3.1lf",
314 isnormal(rt) ? (double)dumpsize / rt : 0.0);
315 errstr = newvstrallocf(errstr, "sec %s kb %s kps %s",
316 walltime_str(runtime), kb_str, kps_str);
317 m = vstrallocf("[%s]", errstr);
320 if(command_in_transit != NULL) {
321 cmdargs = command_in_transit;
322 command_in_transit = NULL;
326 switch(cmdargs->cmd) {
328 putresult(DONE, "%s %lld %s\n", handle,
329 (long long)(dumpsize - (off_t)headersize), q);
330 log_add(L_SUCCESS, "%s %s %s %d [%s]",
331 hostname, qdiskname, chunker_timestamp, level, errstr);
337 if(dumpsize > (off_t)DISK_BLOCK_KB) {
338 putresult(PARTIAL, "%s %lld %s\n", handle,
339 (long long)(dumpsize - (off_t)headersize),
341 log_add(L_PARTIAL, "%s %s %s %d [%s]",
342 hostname, qdiskname, chunker_timestamp, level, errstr);
345 errstr = newvstrallocf(errstr,
346 _("dumper returned %s"), cmdstr[cmdargs->cmd]);
348 m = vstrallocf("[%s]",errstr);
351 putresult(FAILED, "%s %s\n", handle, q);
352 log_add(L_FAIL, "%s %s %s %d [%s]",
353 hostname, qdiskname, chunker_timestamp, level, errstr);
358 } else if(infd != -2) {
360 m = vstrallocf("[%s]", errstr);
365 putresult(FAILED, "%s %s\n", handle, q);
367 log_add(L_FAIL, "%s %s %s %d [%s]",
368 hostname, qdiskname, chunker_timestamp, level, errstr);
376 if(cmdargs->argc >= 1) {
377 q = quote_string(cmdargs->argv[0]);
379 q = stralloc(_("(no input?)"));
381 putresult(BAD_COMMAND, "%s\n", q);
386 /* } while(cmdargs->cmd != QUIT); */
388 log_add(L_INFO, "pid-done %ld", (long)getpid());
391 amfree(chunker_timestamp);
399 free_cmdargs(cmdargs);
400 if (command_in_transit)
401 free_cmdargs(command_in_transit);
402 am_release_feature_set(their_features);
403 their_features = NULL;
407 return (0); /* exit */
411 * Returns a file descriptor to the incoming port
412 * on success, or -1 on error.
422 char *tmp_filename, *pc;
426 struct addrinfo *res;
429 if ((result = resolve_hostname("localhost", 0, &res, NULL) != 0)) {
430 errstr = newvstrallocf(errstr, _("could not resolve localhost: %s"),
431 gai_strerror(result));
434 data_socket = stream_server(res->ai_family, &data_port, 0,
436 if (res) freeaddrinfo(res);
438 if(data_socket < 0) {
439 errstr = vstrallocf(_("error creating stream server: %s"), strerror(errno));
443 putresult(PORT, "%d\n", data_port);
445 infd = stream_accept(data_socket, CONNECT_TIMEOUT, 0, STREAM_BUFSIZE);
448 errstr = vstrallocf(_("error accepting stream: %s"), strerror(errno));
452 tmp_filename = vstralloc(filename, ".tmp", NULL);
453 pc = strrchr(tmp_filename, '/');
454 g_assert(pc != NULL);
456 mkholdingdir(tmp_filename);
458 if ((outfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600)) < 0) {
459 int save_errno = errno;
460 char *m = vstrallocf(_("holding file \"%s\": %s"),
464 errstr = quote_string(m);
466 amfree(tmp_filename);
468 if(save_errno == ENOSPC) {
469 putresult(NO_ROOM, "%s %lld\n",
470 handle, (long long)use);
476 amfree(tmp_filename);
477 databuf_init(db, outfd, filename, use, chunksize);
488 char header_buf[DISK_BLOCK_BYTES];
492 dumpsize = dumpbytes = filesize = (off_t)0;
494 memset(header_buf, 0, sizeof(header_buf));
497 * The first thing we should receive is the file header, which we
498 * need to save into "file", as well as write out. Later, the
499 * chunk code will rewrite it.
501 nread = full_read(infd, header_buf, SIZEOF(header_buf));
502 if (nread != sizeof(header_buf)) {
504 errstr = vstrallocf(_("cannot read header: %s"), strerror(errno));
506 errstr = vstrallocf(_("cannot read header: got %zd bytes instead of %zd"),
507 nread, sizeof(header_buf));
511 parse_file_header(header_buf, &file, (size_t)nread);
512 if(write_tapeheader(db->fd, &file)) {
513 int save_errno = errno;
514 char *m = vstrallocf(_("write_tapeheader file %s: %s"),
515 db->filename, strerror(errno));
516 errstr = quote_string(m);
518 if(save_errno == ENOSPC) {
519 putresult(NO_ROOM, "%s %lld\n", handle,
520 (long long)(db->use+db->split_size-dumpsize));
524 dumpsize += (off_t)DISK_BLOCK_KB;
525 filesize = (off_t)DISK_BLOCK_KB;
526 headersize += DISK_BLOCK_KB;
529 * We've written the file header. Now, just write data until the
532 while ((nread = full_read(infd, db->buf,
533 (size_t)(db->datalimit - db->datain))) > 0) {
535 while(db->dataout < db->datain) {
536 if(!databuf_flush(db)) {
541 while(db->dataout < db->datain) {
542 if(!databuf_flush(db)) {
546 if(dumpbytes > (off_t)0) {
547 dumpsize += (off_t)1; /* count partial final KByte */
548 filesize += (off_t)1;
554 * Initialize a databuf. Takes a writeable file descriptor.
565 db->filename = stralloc(filename);
566 db->filename_seq = (off_t)0;
567 db->chunk_size = chunk_size;
568 db->split_size = (db->chunk_size > use) ? use : db->chunk_size;
569 db->use = (use > db->split_size) ? use - db->split_size : (off_t)0;
570 db->datain = db->dataout = db->buf;
571 db->datalimit = db->buf + SIZEOF(db->buf);
576 * Write out the buffer to the backing file
582 struct cmdargs *cmdargs = NULL;
586 char *arg_filename = NULL;
587 char *new_filename = NULL;
588 char *tmp_filename = NULL;
589 char sequence[NUM_STR_SIZE];
591 filetype_t save_type;
597 * If there's no data, do nothing.
599 if (db->dataout >= db->datain) {
604 * See if we need to split this file.
606 while (db->split_size > (off_t)0 && dumpsize >= db->split_size) {
607 if( db->use == (off_t)0 ) {
609 * Probably no more space on this disk. Request some more.
611 putresult(RQ_MORE_DISK, "%s\n", handle);
613 if(command_in_transit == NULL &&
614 (cmdargs->cmd == DONE || cmdargs->cmd == TRYAGAIN || cmdargs->cmd == FAILED)) {
615 command_in_transit = cmdargs;
618 if(cmdargs->cmd == CONTINUE) {
626 a = 2; /* skip CONTINUE and serial */
628 if(a >= cmdargs->argc) {
629 error(_("error [chunker CONTINUE: not enough args: filename]"));
632 arg_filename = newstralloc(arg_filename, cmdargs->argv[a++]);
634 if(a >= cmdargs->argc) {
635 error(_("error [chunker CONTINUE: not enough args: chunksize]"));
638 db->chunk_size = OFF_T_ATOI(cmdargs->argv[a++]);
639 db->chunk_size = am_floor(db->chunk_size, (off_t)DISK_BLOCK_KB);
641 if(a >= cmdargs->argc) {
642 error(_("error [chunker CONTINUE: not enough args: use]"));
645 db->use = OFF_T_ATOI(cmdargs->argv[a++]);
647 if(a != cmdargs->argc) {
648 error(_("error [chunker CONTINUE: too many args: %d != %d]"),
653 if(strcmp(db->filename, arg_filename) == 0) {
655 * Same disk, so use what room is left up to the
656 * next chunk boundary or the amount we were given,
659 left_in_chunk = db->chunk_size - filesize;
660 if(left_in_chunk > db->use) {
661 db->split_size += db->use;
664 db->split_size += left_in_chunk;
665 db->use -= left_in_chunk;
667 if(left_in_chunk > (off_t)0) {
669 * We still have space in this chunk.
675 * Different disk, so use new file.
677 db->filename = newstralloc(db->filename, arg_filename);
679 } else if(cmdargs->cmd == ABORT) {
681 errstr = newstralloc(errstr, cmdargs->argv[1]);
682 putresult(ABORT_FINISHED, "%s\n", handle);
686 if(cmdargs->argc >= 1) {
687 q = quote_string(cmdargs->argv[0]);
689 q = stralloc(_("(no input?)"));
691 error(_("error [bad command after RQ-MORE-DISK: \"%s\"]"), q);
697 * Time to use another file.
701 * First, open the new chunk file, and give it a new header
702 * that has no cont_filename pointer.
704 g_snprintf(sequence, SIZEOF(sequence), "%d", db->filename_seq);
705 new_filename = newvstralloc(new_filename,
710 tmp_filename = newvstralloc(tmp_filename,
714 pc = strrchr(tmp_filename, '/');
715 g_assert(pc != NULL); /* Only a problem if db->filename has no /. */
717 mkholdingdir(tmp_filename);
719 newfd = open(tmp_filename, O_RDWR|O_CREAT|O_TRUNC, 0600);
721 int save_errno = errno;
724 if(save_errno == ENOSPC) {
725 putresult(NO_ROOM, "%s %lld\n", handle,
726 (long long)(db->use+db->split_size-dumpsize));
727 db->use = (off_t)0; /* force RQ_MORE_DISK */
728 db->split_size = dumpsize;
731 m = vstrallocf(_("creating chunk holding file \"%s\": %s"),
734 errstr = quote_string(m);
740 save_type = file.type;
741 file.type = F_CONT_DUMPFILE;
742 file.cont_filename[0] = '\0';
743 if(write_tapeheader(newfd, &file)) {
744 int save_errno = errno;
748 if(save_errno == ENOSPC) {
749 putresult(NO_ROOM, "%s %lld\n", handle,
750 (long long)(db->use+db->split_size-dumpsize));
751 db->use = (off_t)0; /* force RQ_MORE DISK */
752 db->split_size = dumpsize;
755 m = vstrallocf(_("write_tapeheader file %s: %s"),
758 errstr = quote_string(m);
765 * Now, update the header of the current file to point
766 * to the next chunk, and then close it.
768 if (lseek(db->fd, (off_t)0, SEEK_SET) < (off_t)0) {
769 char *m = vstrallocf(_("lseek holding file %s: %s"),
772 errstr = quote_string(m);
779 file.type = save_type;
780 strncpy(file.cont_filename, new_filename, SIZEOF(file.cont_filename));
781 file.cont_filename[SIZEOF(file.cont_filename)-1] = '\0';
782 if(write_tapeheader(db->fd, &file)) {
783 char * m = vstrallocf(_("write_tapeheader file \"%s\": %s"),
786 errstr = quote_string(m);
789 unlink(tmp_filename);
793 file.type = F_CONT_DUMPFILE;
796 * Now shift the file descriptor.
803 * Update when we need to chunk again
805 if(db->use <= (off_t)DISK_BLOCK_KB) {
807 * Cheat and use one more block than allowed so we can make
810 db->split_size += (off_t)(2 * DISK_BLOCK_KB);
812 } else if(db->chunk_size > db->use) {
813 db->split_size += db->use;
816 db->split_size += db->chunk_size;
817 db->use -= db->chunk_size;
821 amfree(tmp_filename);
822 amfree(new_filename);
823 dumpsize += (off_t)DISK_BLOCK_KB;
824 filesize = (off_t)DISK_BLOCK_KB;
825 headersize += DISK_BLOCK_KB;
830 * Write out the buffer
832 written = full_write(db->fd, db->dataout,
833 (size_t)(db->datain - db->dataout));
835 db->dataout += written;
836 dumpbytes += (off_t)written;
838 dumpsize += (dumpbytes / (off_t)1024);
839 filesize += (dumpbytes / (off_t)1024);
842 if (errno != ENOSPC) {
843 char *m = vstrallocf(_("data write: %s"), strerror(errno));
844 errstr = quote_string(m);
851 * NO-ROOM is informational only. Later, RQ_MORE_DISK will be
852 * issued to use another holding disk.
854 putresult(NO_ROOM, "%s %lld\n", handle,
855 (long long)(db->use+db->split_size-dumpsize));
856 db->use = (off_t)0; /* force RQ_MORE_DISK */
857 db->split_size = dumpsize;
860 if (db->datain == db->dataout) {
862 * We flushed the whole buffer so reset to use it all.
864 db->datain = db->dataout = db->buf;
870 free_cmdargs(cmdargs);
871 amfree(new_filename);
872 /*@i@*/ amfree(tmp_filename);
873 amfree(arg_filename);
879 * Send an Amanda dump header to the output file and set file->blocksize
889 file->blocksize = DISK_BLOCK_BYTES;
890 buffer = build_header(file, DISK_BLOCK_BYTES);
892 written = full_write(outfd, buffer, DISK_BLOCK_BYTES);
894 if(written == DISK_BLOCK_BYTES) return 0;
896 /* fake ENOSPC when we get a short write without errno set */