2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1998 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
27 * $Id: driver.c 6512 2007-05-24 17:00:24Z ian $
29 * controlling process for the Amanda backup system
33 * XXX possibly modify tape queue to be cognizant of how much room is left on
34 * tape. Probably not effective though, should do this in planner.
48 #include "server_util.h"
49 #include "timestamp.h"
51 #define driver_debug(i, ...) do { \
52 if ((i) <= debug_driver) { \
53 dbprintf(__VA_ARGS__); \
57 #define hold_debug(i, ...) do { \
58 if ((i) <= debug_holding) { \
59 dbprintf(__VA_ARGS__); \
63 static disklist_t waitq; // dle waiting estimate result
64 static disklist_t runq; // dle waiting to be dumped to holding disk
65 static disklist_t directq; // dle waiting to be dumped directly to tape
66 static disklist_t tapeq; // dle on holding disk waiting to be written
68 static disklist_t roomq; // dle waiting for more space on holding disk
69 static int pending_aborts;
70 static int degraded_mode;
71 static off_t reserved_space;
72 static off_t total_disksize;
73 static char *dumper_program;
74 static char *chunker_program;
75 static int inparallel;
76 static int nodump = 0;
77 static off_t tape_length = (off_t)0;
78 static int current_tape = 0;
79 static int conf_taperalgo;
80 static int conf_taper_parallel_write;
81 static int conf_runtapes;
82 static time_t sleep_time;
83 static int idle_reason;
84 static char *driver_timestamp;
85 static char *hd_driver_timestamp;
86 static am_host_t *flushhost = NULL;
87 static int need_degraded=0;
88 static holdalloc_t *holdalloc;
89 static int num_holdalloc;
90 static event_handle_t *dumpers_ev_time = NULL;
91 static event_handle_t *flush_ev_read = NULL;
92 static event_handle_t *schedule_ev_read = NULL;
93 static int conf_flush_threshold_dumped;
94 static int conf_flush_threshold_scheduled;
95 static int conf_taperflush;
96 static off_t flush_threshold_dumped;
97 static off_t flush_threshold_scheduled;
98 static off_t taperflush;
99 static int schedule_done; // 1 if we don't wait for a
100 // schedule from the planner
101 static int force_flush; // All dump are terminated, we
102 // must now respect taper_flush
103 static int taper_nb_scan_volume = 0;
104 static int nb_sent_new_tape = 0;
105 static int taper_started = 0;
106 static taper_t *last_started_taper;
108 static int wait_children(int count);
109 static void wait_for_children(void);
110 static void allocate_bandwidth(netif_t *ip, unsigned long kps);
111 static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp);
112 static void adjust_diskspace(disk_t *diskp, cmd_t cmd);
113 static void delete_diskspace(disk_t *diskp);
114 static assignedhd_t **build_diskspace(char *destname);
115 static int client_constrained(disk_t *dp);
116 static void deallocate_bandwidth(netif_t *ip, unsigned long kps);
117 static void dump_schedule(disklist_t *qp, char *str);
118 static assignedhd_t **find_diskspace(off_t size, int *cur_idle,
119 assignedhd_t *preferred);
120 static unsigned long free_kps(netif_t *ip);
121 static off_t free_space(void);
122 static void dumper_chunker_result(disk_t *dp);
123 static void dumper_taper_result(disk_t *dp);
124 static void file_taper_result(disk_t *dp);
125 static void handle_dumper_result(void *);
126 static void handle_chunker_result(void *);
127 static void handle_dumpers_time(void *);
128 static void handle_taper_result(void *);
130 static void holdingdisk_state(char *time_str);
131 static taper_t *idle_taper(void);
132 static taper_t *taper_from_name(char *name);
133 static void interface_state(char *time_str);
134 static int queue_length(disklist_t q);
135 static void read_flush(void *cookie);
136 static void read_schedule(void *cookie);
137 static void short_dump_state(void);
138 static void startaflush(void);
139 static void start_degraded_mode(disklist_t *queuep);
140 static void start_some_dumps(disklist_t *rq);
141 static void continue_port_dumps(void);
142 static void update_failed_dump(disk_t *);
143 static int no_taper_flushing(void);
146 TAPE_ACTION_NO_ACTION = 0,
147 TAPE_ACTION_SCAN = (1 << 0),
148 TAPE_ACTION_NEW_TAPE = (1 << 1),
149 TAPE_ACTION_NO_NEW_TAPE = (1 << 2),
150 TAPE_ACTION_START_A_FLUSH = (1 << 3),
151 TAPE_ACTION_START_A_FLUSH_FIT = (1 << 4),
152 TAPE_ACTION_MOVE = (1 << 5)
155 static TapeAction tape_action(taper_t *taper, char **why_no_new_tape);
157 static const char *idle_strings[] = {
160 #define IDLE_NO_DUMPERS 1
162 #define IDLE_START_WAIT 2
164 #define IDLE_NO_HOLD 3
166 #define IDLE_CLIENT_CONSTRAINED 4
167 T_("client-constrained"),
168 #define IDLE_NO_BANDWIDTH 5
170 #define IDLE_NO_DISKSPACE 6
184 struct fs_usage fsusage;
187 unsigned long reserve = 100;
189 char **result_argv = NULL;
196 config_overrides_t *cfg_ovr = NULL;
197 char *cfg_opt = NULL;
198 holdalloc_t *ha, *ha_last;
199 find_result_t *holding_files;
200 disklist_t holding_disklist = { NULL, NULL };
201 int no_taper = FALSE;
202 int from_client = FALSE;
205 * Configure program for internationalization:
206 * 1) Only set the message locale for now.
207 * 2) Set textdomain for all amanda related programs to "amanda"
208 * We don't want to be forced to support dozens of message catalogs.
210 setlocale(LC_MESSAGES, "C");
211 textdomain("amanda");
215 setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0);
216 setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0);
220 dbopen(DBG_SUBDIR_SERVER);
222 atexit(wait_for_children);
224 /* Don't die when child closes pipe */
225 signal(SIGPIPE, SIG_IGN);
227 add_amanda_log_handler(amanda_log_stderr);
228 add_amanda_log_handler(amanda_log_trace_log);
232 cfg_ovr = extract_commandline_config_overrides(&argc, &argv);
236 set_config_overrides(cfg_ovr);
237 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
239 conf_diskfile = config_dir_relative(getconf_str(CNF_DISKFILE));
240 read_diskfile(conf_diskfile, &origq);
241 disable_skip_disk(&origq);
242 amfree(conf_diskfile);
244 if (config_errors(NULL) >= CFGERR_WARNINGS) {
245 config_print_errors();
246 if (config_errors(NULL) >= CFGERR_ERRORS) {
247 g_critical(_("errors processing config file"));
251 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
252 g_printf(_("%s: pid %ld executable %s version %s\n"),
253 get_pname(), (long) getpid(), argv[0], VERSION);
256 if(strcmp(argv[2], "nodump") == 0) {
264 if (strcmp(argv[2], "--no-taper") == 0) {
272 if (strcmp(argv[2], "--from-client") == 0) {
279 safe_cd(); /* do this *after* config_init */
281 check_running_as(RUNNING_AS_DUMPUSER);
283 dbrename(get_config_name(), DBG_SUBDIR_SERVER);
285 /* load DLEs from the holding disk, in case there's anything to flush there */
286 search_holding_disk(&holding_files, &holding_disklist);
287 /* note that the dumps are added to the global disklist, so we need not consult
288 * holding_files or holding_disklist after this */
290 amfree(driver_timestamp);
291 /* read timestamp from stdin */
292 while ((line = agets(stdin)) != NULL) {
297 if ( line == NULL ) {
298 error(_("Did not get DATE line from planner"));
301 driver_timestamp = alloc(15);
302 strncpy(driver_timestamp, &line[5], 14);
303 driver_timestamp[14] = '\0';
305 log_add(L_START,_("date %s"), driver_timestamp);
307 gethostname(hostname, SIZEOF(hostname));
308 log_add(L_STATS,_("hostname %s"), hostname);
310 /* check that we don't do many dump in a day and usetimestamps is off */
311 if(strlen(driver_timestamp) == 8) {
313 char *conf_logdir = getconf_str(CNF_LOGDIR);
314 char *logfile = vstralloc(conf_logdir, "/log.",
315 driver_timestamp, ".0", NULL);
316 char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.",
317 driver_timestamp, ".0", NULL);
318 if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) {
319 log_add(L_WARNING, _("WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day."));
324 hd_driver_timestamp = get_timestamp_from_time(0);
327 hd_driver_timestamp = stralloc(driver_timestamp);
330 taper_program = vstralloc(amlibexecdir, "/", "taper", NULL);
331 dumper_program = vstralloc(amlibexecdir, "/", "dumper", NULL);
332 chunker_program = vstralloc(amlibexecdir, "/", "chunker", NULL);
334 conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO);
335 conf_taper_parallel_write = getconf_int(CNF_TAPER_PARALLEL_WRITE);
336 conf_tapetype = getconf_str(CNF_TAPETYPE);
337 conf_runtapes = getconf_int(CNF_RUNTAPES);
338 if (conf_taper_parallel_write > conf_runtapes) {
339 conf_taper_parallel_write = conf_runtapes;
341 tape = lookup_tapetype(conf_tapetype);
342 tape_length = tapetype_get_length(tape);
343 g_printf("driver: tape size %lld\n", (long long)tape_length);
344 conf_flush_threshold_dumped = getconf_int(CNF_FLUSH_THRESHOLD_DUMPED);
345 conf_flush_threshold_scheduled = getconf_int(CNF_FLUSH_THRESHOLD_SCHEDULED);
346 conf_taperflush = getconf_int(CNF_TAPERFLUSH);
348 flush_threshold_dumped = (conf_flush_threshold_dumped * tape_length) / 100;
349 flush_threshold_scheduled = (conf_flush_threshold_scheduled * tape_length) / 100;
350 taperflush = (conf_taperflush *tape_length) / 100;
352 driver_debug(1, _("flush-threshold-dumped: %lld\n"), (long long)flush_threshold_dumped);
353 driver_debug(1, _("flush-threshold-scheduled: %lld\n"), (long long)flush_threshold_scheduled);
354 driver_debug(1, _("taperflush: %lld\n"), (long long)taperflush);
356 /* set up any configuration-dependent variables */
358 inparallel = getconf_int(CNF_INPARALLEL);
360 reserve = (unsigned long)getconf_int(CNF_RESERVE);
362 total_disksize = (off_t)0;
365 for (il = getconf_identlist(CNF_HOLDINGDISK), dsk = 0;
367 il = il->next, dsk++) {
368 hdp = lookup_holdingdisk(il->data);
369 ha = alloc(SIZEOF(holdalloc_t));
372 /* link the list in the same order as getconf_holdingdisks's results */
381 ha->allocated_dumpers = 0;
382 ha->allocated_space = (off_t)0;
383 ha->disksize = holdingdisk_get_disksize(hdp);
386 if(get_fs_usage(holdingdisk_get_diskdir(hdp), NULL, &fsusage) == -1
387 || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) {
388 log_add(L_WARNING, _("WARNING: ignoring holding disk %s: %s\n"),
389 holdingdisk_get_diskdir(hdp), strerror(errno));
394 /* do the division first to avoid potential integer overflow */
395 if (fsusage.fsu_bavail_top_bit_set)
398 kb_avail = fsusage.fsu_bavail / 1024 * fsusage.fsu_blocksize;
400 if(ha->disksize > (off_t)0) {
401 if(ha->disksize > kb_avail) {
403 _("WARNING: %s: %lld KB requested, "
404 "but only %lld KB available."),
405 holdingdisk_get_diskdir(hdp),
406 (long long)ha->disksize,
407 (long long)kb_avail);
408 ha->disksize = kb_avail;
411 /* ha->disksize is negative; use all but that amount */
412 else if(kb_avail < -ha->disksize) {
414 _("WARNING: %s: not %lld KB free."),
415 holdingdisk_get_diskdir(hdp),
416 (long long)-ha->disksize);
417 ha->disksize = (off_t)0;
421 ha->disksize += kb_avail;
423 g_printf(_("driver: adding holding disk %d dir %s size %lld chunksize %lld\n"),
424 dsk, holdingdisk_get_diskdir(hdp),
425 (long long)ha->disksize,
426 (long long)(holdingdisk_get_chunksize(hdp)));
428 newdir = newvstralloc(newdir,
429 holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp,
431 if(!mkholdingdir(newdir)) {
432 ha->disksize = (off_t)0;
434 total_disksize += ha->disksize;
437 reserved_space = total_disksize * (off_t)(reserve / 100);
439 g_printf(_("reserving %lld out of %lld for degraded-mode dumps\n"),
440 (long long)reserved_space, (long long)free_space());
444 if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS;
446 /* taper takes a while to get going, so start it up right away */
449 startup_tape_process(taper_program, conf_taper_parallel_write, no_taper);
451 /* fire up the dumpers now while we are waiting */
452 if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp);
455 * Read schedule from stdin. Usually, this is a pipe from planner,
456 * so the effect is that we wait here for the planner to
457 * finish, but meanwhile the taper is rewinding the tape, reading
458 * the label, checking it, writing a new label and all that jazz
459 * in parallel with the planner.
471 taper_nb_wait_reply = 0;
474 if (no_taper || conf_runtapes <= 0) {
475 taper_started = 1; /* we'll pretend the taper started and failed immediately */
478 tapetable[0].state = TAPER_STATE_INIT;
479 taper_nb_wait_reply++;
480 taper_nb_scan_volume++;
481 taper_ev_read = event_register(taper_fd, EV_READFD,
482 handle_taper_result, NULL);
483 taper_cmd(START_TAPER, NULL, tapetable[0].name, 0, driver_timestamp);
486 flush_ev_read = event_register((event_id_t)0, EV_READFD, read_flush, NULL);
488 log_add(L_STATS, _("startup time %s"), walltime_str(curclock()));
490 g_printf(_("driver: start time %s inparallel %d bandwidth %lu diskspace %lld "), walltime_str(curclock()), inparallel,
491 free_kps(NULL), (long long)free_space());
492 g_printf(_(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n"),
493 "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo),
494 getconf_str(CNF_DUMPORDER));
497 schedule_done = nodump;
505 /* mv runq to directq */
506 while (!empty(runq)) {
507 diskp = dequeue_disk(&runq);
508 headqueue_disk(&directq, diskp);
511 run_server_global_scripts(EXECUTE_ON_POST_BACKUP, get_config_name());
513 /* log error for any remaining dumps */
514 while(!empty(directq)) {
515 diskp = dequeue_disk(&directq);
517 if (diskp->to_holdingdisk == HOLD_REQUIRED) {
518 char *qname = quote_string(diskp->name);
519 log_add(L_FAIL, "%s %s %s %d [%s]",
520 diskp->host->hostname, qname, sched(diskp)->datestamp,
522 _("can't dump required holdingdisk"));
525 else if (!degraded_mode) {
526 char *qname = quote_string(diskp->name);
527 log_add(L_FAIL, "%s %s %s %d [%s]",
528 diskp->host->hostname, qname, sched(diskp)->datestamp,
530 _("can't dump in non degraded mode"));
534 char *qname = quote_string(diskp->name);
535 log_add(L_FAIL, "%s %s %s %d [%s]",
536 diskp->host->hostname, qname, sched(diskp)->datestamp,
539 _("can't do degraded dump without holding disk") :
540 diskp->to_holdingdisk != HOLD_NEVER ?
541 _("out of holding space in degraded mode") :
542 _("can't dump 'holdingdisk never' dle in degraded mode"));
547 short_dump_state(); /* for amstatus */
549 g_printf(_("driver: QUITTING time %s telling children to quit\n"),
550 walltime_str(curclock()));
554 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
556 dumper_cmd(dumper, QUIT, NULL, NULL);
561 taper_cmd(QUIT, NULL, NULL, 0, NULL);
564 /* wait for all to die */
568 holding_cleanup(NULL, NULL);
572 check_unfree_serial();
573 g_printf(_("driver: FINISHED time %s\n"), walltime_str(curclock()));
575 log_add(L_FINISH,_("date %s time %s"), driver_timestamp, walltime_str(curclock()));
576 log_add(L_INFO, "pid-done %ld", (long)getpid());
577 amfree(driver_timestamp);
579 amfree(dumper_program);
580 amfree(taper_program);
582 g_strfreev(result_argv);
589 /* sleep up to count seconds, and wait for terminating child process */
590 /* if sleep is negative, this function will not timeout */
591 /* exit once all child process are finished or the timout expired */
592 /* return 0 if no more children to wait */
593 /* return 1 if some children are still alive */
595 wait_children(int count)
607 pid = waitpid((pid_t)-1, &retstat, WNOHANG);
611 if (! WIFEXITED(retstat)) {
613 code = WTERMSIG(retstat);
614 } else if (WEXITSTATUS(retstat) != 0) {
616 code = WEXITSTATUS(retstat);
619 for (dumper = dmptable; dumper < dmptable + inparallel;
621 if (pid == dumper->pid) {
622 who = stralloc(dumper->name);
626 if (dumper->chunker && pid == dumper->chunker->pid) {
627 who = stralloc(dumper->chunker->name);
628 dumper->chunker->pid = -1;
632 if (who == NULL && pid == taper_pid) {
633 who = stralloc("taper");
636 if(what != NULL && who == NULL) {
637 who = stralloc("unknown");
640 log_add(L_WARNING, _("%s pid %u exited with %s %d\n"), who,
641 (unsigned)pid, what, code);
642 g_printf(_("driver: %s pid %u exited with %s %d\n"), who,
643 (unsigned)pid, what, code);
647 } while (pid > 0 || wait_errno == EINTR);
652 } while ((errno != ECHILD) && (count != 0));
653 return (errno != ECHILD);
657 kill_children(int signal)
662 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
663 if (!dumper->down && dumper->pid > 1) {
664 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
665 dumper->name, (unsigned)dumper->pid);
666 if (kill(dumper->pid, signal) == -1 && errno == ESRCH) {
668 dumper->chunker->pid = 0;
670 if (dumper->chunker && dumper->chunker->pid > 1) {
671 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
672 dumper->chunker->name,
673 (unsigned)dumper->chunker->pid);
674 if (kill(dumper->chunker->pid, signal) == -1 &&
676 dumper->chunker->pid = 0;
683 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
684 "taper", (unsigned)taper_pid);
685 if (kill(taper_pid, signal) == -1 && errno == ESRCH)
691 wait_for_children(void)
696 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
697 if (dumper->pid > 1 && dumper->fd >= 0) {
698 dumper_cmd(dumper, QUIT, NULL, NULL);
699 if (dumper->chunker && dumper->chunker->pid > 1 &&
700 dumper->chunker->fd >= 0)
701 chunker_cmd(dumper->chunker, QUIT, NULL, NULL);
706 if(taper_pid > 1 && taper_fd > 0) {
707 taper_cmd(QUIT, NULL, NULL, 0, NULL);
710 if(wait_children(60) == 0)
713 kill_children(SIGHUP);
714 if(wait_children(60) == 0)
717 kill_children(SIGKILL);
718 if(wait_children(-1) == 0)
723 static void startaflush_tape(taper_t *taper);
730 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
732 if (!(taper->state & TAPER_STATE_DONE) &&
733 taper->state & TAPER_STATE_WAIT_FOR_TAPE) {
734 startaflush_tape(taper);
737 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
739 if (!(taper->state & TAPER_STATE_DONE) &&
740 taper->state & TAPER_STATE_TAPE_REQUESTED) {
741 startaflush_tape(taper);
744 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
746 if (!(taper->state & TAPER_STATE_DONE) &&
747 taper->state & TAPER_STATE_INIT) {
748 startaflush_tape(taper);
751 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
753 if (!(taper->state & TAPER_STATE_DONE) &&
754 taper->state & TAPER_STATE_IDLE) {
755 startaflush_tape(taper);
767 off_t extra_tapes_size = 0;
770 TapeAction result_tape_action;
771 char *why_no_new_tape = NULL;
774 result_tape_action = tape_action(taper, &why_no_new_tape);
776 if (result_tape_action & TAPE_ACTION_SCAN) {
777 taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
778 taper->state |= TAPER_STATE_WAIT_FOR_TAPE;
779 taper_nb_scan_volume++;
780 taper_cmd(START_SCAN, taper->disk, NULL, 0, NULL);
781 } else if (result_tape_action & TAPE_ACTION_NEW_TAPE) {
782 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
783 taper->state |= TAPER_STATE_WAIT_NEW_TAPE;
785 taper_cmd(NEW_TAPE, taper->disk, NULL, 0, NULL);
786 } else if (result_tape_action & TAPE_ACTION_NO_NEW_TAPE) {
787 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
788 taper_cmd(NO_NEW_TAPE, taper->disk, why_no_new_tape, 0, NULL);
789 taper->state |= TAPER_STATE_DONE;
790 start_degraded_mode(&runq);
791 } else if (result_tape_action & TAPE_ACTION_MOVE) {
792 taper_t *taper1 = idle_taper();
794 taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
795 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
796 taper_cmd(TAKE_SCRIBE_FROM, taper->disk, taper1->name, 0 , NULL);
797 taper1->state = TAPER_STATE_DEFAULT;
798 taper->state |= TAPER_STATE_TAPE_STARTED;
799 taper->left = taper1->left;
800 if (last_started_taper == taper1) {
801 last_started_taper = taper;
806 if (!degraded_mode &&
807 taper->state & TAPER_STATE_IDLE &&
809 (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
810 result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT)) {
812 int taperalgo = conf_taperalgo;
813 if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
814 if (taperalgo == ALGO_FIRST)
815 taperalgo = ALGO_FIRSTFIT;
816 else if (taperalgo == ALGO_LARGEST)
817 taperalgo = ALGO_LARGESTFIT;
818 else if (taperalgo == ALGO_SMALLEST)
819 taperalgo = ALGO_SMALLESTFIT;
820 else if (taperalgo == ALGO_LAST)
821 taperalgo = ALGO_LASTFIT;
824 extra_tapes_size = tape_length * (off_t)(conf_runtapes - current_tape);
825 for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
827 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
828 extra_tapes_size += taper1->left;
832 extra_tapes_size -= (sched(dp)->act_size - taper1->written);
836 if (taper->state & TAPER_STATE_TAPE_STARTED) {
837 taper_left = taper->left;
839 taper_left = tape_length;
842 datestamp = sched(tapeq.head)->datestamp;
845 dp = dequeue_disk(&tapeq);
849 while (fit != NULL) {
850 if (sched(fit)->act_size <=
851 (fit->splitsize ? extra_tapes_size : taper_left) &&
852 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
860 if(dp) remove_disk(&tapeq, dp);
863 fit = dp = tapeq.head;
864 while (fit != NULL) {
865 if(sched(fit)->act_size > sched(dp)->act_size &&
866 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
871 if(dp) remove_disk(&tapeq, dp);
873 case ALGO_LARGESTFIT:
875 while (fit != NULL) {
876 if(sched(fit)->act_size <=
877 (fit->splitsize ? extra_tapes_size : taper_left) &&
878 (!dp || sched(fit)->act_size > sched(dp)->act_size) &&
879 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
884 if(dp) remove_disk(&tapeq, dp);
887 fit = dp = tapeq.head;
888 while (fit != NULL) {
889 if (sched(fit)->act_size < sched(dp)->act_size &&
890 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
895 if(dp) remove_disk(&tapeq, dp);
897 case ALGO_SMALLESTFIT:
898 fit = dp = tapeq.head;
899 while (fit != NULL) {
900 if (sched(fit)->act_size <=
901 (fit->splitsize ? extra_tapes_size : taper_left) &&
902 (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
903 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
908 if(dp) remove_disk(&tapeq, dp);
912 remove_disk(&tapeq, dp);
916 while (fit != NULL) {
917 if (sched(fit)->act_size <=
918 (fit->splitsize ? extra_tapes_size : taper_left) &&
919 (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
920 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
925 if(dp) remove_disk(&tapeq, dp);
929 if (!(result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT)) {
930 if(conf_taperalgo != ALGO_SMALLEST) {
932 _("driver: startaflush: Using SMALLEST because nothing fit\n"));
935 fit = dp = tapeq.head;
936 while (fit != NULL) {
937 if (sched(fit)->act_size < sched(dp)->act_size &&
938 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
943 if(dp) remove_disk(&tapeq, dp);
948 taper->dumper = NULL;
949 amfree(taper->input_error);
950 amfree(taper->tape_error);
951 taper->result = LAST_TOK;
952 taper->sendresult = 0;
953 amfree(taper->first_label);
955 taper->state &= ~TAPER_STATE_IDLE;
956 taper->state |= TAPER_STATE_FILE_TO_TAPE;
957 taper->dumper = NULL;
958 qname = quote_string(dp->name);
959 if (taper_nb_wait_reply == 0) {
960 taper_ev_read = event_register(taper_fd, EV_READFD,
961 handle_taper_result, NULL);
963 taper_nb_wait_reply++;
964 sched(dp)->taper = taper;
965 taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level,
966 sched(dp)->datestamp);
967 g_fprintf(stderr,_("driver: startaflush: %s %s %s %lld %lld\n"),
968 taperalgo2str(taperalgo), dp->host->hostname, qname,
969 (long long)sched(taper->disk)->act_size,
970 (long long)taper->left);
983 /* first, check if host is too busy */
985 if(dp->host->inprogress >= dp->host->maxdumps) {
989 /* next, check conflict with other dumps on same spindle */
991 if(dp->spindle == -1) { /* but spindle -1 never conflicts by def. */
995 for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext)
996 if(dp2->inprogress && dp2->spindle == dp->spindle) {
1010 int dumper_to_holding,
1012 disk_t **delayed_diskp,
1013 disk_t **diskp_accept,
1014 assignedhd_t ***holdp_accept,
1015 off_t extra_tapes_size)
1017 assignedhd_t **holdp=NULL;
1019 if (diskp->host->start_t > now) {
1020 *cur_idle = max(*cur_idle, IDLE_START_WAIT);
1021 if (*delayed_diskp == NULL || sleep_time > diskp->host->start_t) {
1022 *delayed_diskp = diskp;
1023 sleep_time = diskp->host->start_t;
1025 } else if(diskp->start_t > now) {
1026 *cur_idle = max(*cur_idle, IDLE_START_WAIT);
1027 if (*delayed_diskp == NULL || sleep_time > diskp->start_t) {
1028 *delayed_diskp = diskp;
1029 sleep_time = diskp->start_t;
1031 } else if (diskp->host->netif->curusage > 0 &&
1032 sched(diskp)->est_kps > free_kps(diskp->host->netif)) {
1033 *cur_idle = max(*cur_idle, IDLE_NO_BANDWIDTH);
1034 } else if (!taper && sched(diskp)->no_space) {
1035 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1036 } else if (!taper && diskp->to_holdingdisk == HOLD_NEVER) {
1037 *cur_idle = max(*cur_idle, IDLE_NO_HOLD);
1038 } else if (extra_tapes_size && sched(diskp)->est_size > extra_tapes_size) {
1039 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1041 } else if (!taper && (holdp =
1042 find_diskspace(sched(diskp)->est_size, cur_idle, NULL)) == NULL) {
1043 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1044 if (empty(tapeq) && dumper_to_holding == 0 && rq != &directq && no_taper_flushing()) {
1045 remove_disk(rq, diskp);
1046 if (diskp->to_holdingdisk != HOLD_REQUIRED) {
1047 enqueue_disk(&directq, diskp);
1048 diskp->to_holdingdisk = HOLD_NEVER;
1050 if (empty(*rq)) force_flush = 1;
1052 } else if (client_constrained(diskp)) {
1053 free_assignedhd(holdp);
1054 *cur_idle = max(*cur_idle, IDLE_CLIENT_CONSTRAINED);
1057 /* disk fits, dump it */
1058 int accept = !*diskp_accept;
1061 case 's': accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1063 case 'S': accept = (sched(diskp)->est_size > sched(*diskp_accept)->est_size);
1065 case 't': accept = (sched(diskp)->est_time < sched(*diskp_accept)->est_time);
1067 case 'T': accept = (sched(diskp)->est_time > sched(*diskp_accept)->est_time);
1069 case 'b': accept = (sched(diskp)->est_kps < sched(*diskp_accept)->est_kps);
1071 case 'B': accept = (sched(diskp)->est_kps > sched(*diskp_accept)->est_kps);
1073 default: log_add(L_WARNING, _("Unknown dumporder character \'%c\', using 's'.\n"),
1075 accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1080 if( !*diskp_accept || !degraded_mode || diskp->priority >= (*diskp_accept)->priority) {
1081 if(*holdp_accept) free_assignedhd(*holdp_accept);
1082 *diskp_accept = diskp;
1083 *holdp_accept = holdp;
1086 free_assignedhd(holdp);
1090 free_assignedhd(holdp);
1099 const time_t now = time(NULL);
1101 disk_t *diskp, *delayed_diskp, *diskp_accept, *diskp_next;
1103 assignedhd_t **holdp=NULL, **holdp_accept;
1112 int dumper_to_holding = 0;
1114 /* don't start any actual dumps until the taper is started */
1115 if (!taper_started) return;
1117 idle_reason = IDLE_NO_DUMPERS;
1120 if(dumpers_ev_time != NULL) {
1121 event_release(dumpers_ev_time);
1122 dumpers_ev_time = NULL;
1125 for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1126 if (dumper->busy && dumper->dp->to_holdingdisk != HOLD_NEVER) {
1127 dumper_to_holding++;
1130 for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) {
1132 if( dumper->busy || dumper->down) {
1136 if (dumper->ev_read != NULL) {
1137 event_release(dumper->ev_read);
1138 dumper->ev_read = NULL;
1142 * A potential problem with starting from the bottom of the dump time
1143 * distribution is that a slave host will have both one of the shortest
1144 * and one of the longest disks, so starting its shortest disk first will
1145 * tie up the host and eliminate its longest disk from consideration the
1146 * first pass through. This could cause a big delay in starting that long
1147 * disk, which could drag out the whole night's dumps.
1149 * While starting from the top of the dump time distribution solves the
1150 * above problem, this turns out to be a bad idea, because the big dumps
1151 * will almost certainly pack the holding disk completely, leaving no
1152 * room for even one small dump to start. This ends up shutting out the
1153 * small-end dumpers completely (they stay idle).
1155 * The introduction of multiple simultaneous dumps to one host alleviates
1156 * the biggest&smallest dumps problem: both can be started at the
1160 diskp_accept = NULL;
1161 holdp_accept = NULL;
1162 delayed_diskp = NULL;
1164 cur_idle = NOT_IDLE;
1166 dumporder = getconf_str(CNF_DUMPORDER);
1167 if(strlen(dumporder) > (size_t)(dumper-dmptable)) {
1168 dumptype = dumporder[dumper-dmptable];
1171 if(dumper-dmptable < 3)
1179 if (!empty(directq)) {
1180 taper = idle_taper();
1182 TapeAction result_tape_action;
1183 char *why_no_new_tape = NULL;
1184 result_tape_action = tape_action(taper, &why_no_new_tape);
1185 if (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
1186 result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1187 off_t extra_tapes_size = 0;
1190 if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1191 extra_tapes_size = tape_length *
1192 (off_t)(conf_runtapes - current_tape);
1193 for (taper1 = tapetable;
1194 taper1 < tapetable + conf_taper_parallel_write;
1196 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
1197 extra_tapes_size += taper1->left;
1201 extra_tapes_size -= (sched(dp)->est_size -
1207 for (diskp = directq.head; diskp != NULL;
1208 diskp = diskp_next) {
1209 diskp_next = diskp->next;
1210 allow_dump_dle(diskp, taper, dumptype, &directq, now,
1211 dumper_to_holding, &cur_idle,
1212 &delayed_diskp, &diskp_accept,
1213 &holdp_accept, extra_tapes_size);
1216 diskp = diskp_accept;
1217 holdp = holdp_accept;
1227 if (diskp == NULL) {
1228 for(diskp = rq->head; diskp != NULL; diskp = diskp_next) {
1229 diskp_next = diskp->next;
1230 assert(diskp->host != NULL && sched(diskp) != NULL);
1232 allow_dump_dle(diskp, NULL, dumptype, rq, now,
1233 dumper_to_holding, &cur_idle, &delayed_diskp,
1234 &diskp_accept, &holdp_accept, 0);
1236 diskp = diskp_accept;
1237 holdp = holdp_accept;
1240 idle_reason = max(idle_reason, cur_idle);
1241 if (diskp == NULL && idle_reason == IDLE_NO_DISKSPACE) {
1242 /* continue flush waiting for new tape */
1247 * If we have no disk at this point, and there are disks that
1248 * are delayed, then schedule a time event to call this dumper
1249 * with the disk with the shortest delay.
1251 if (diskp == NULL && delayed_diskp != NULL) {
1252 assert(sleep_time > now);
1254 dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME,
1255 handle_dumpers_time, &runq);
1257 } else if (diskp != NULL && taper == NULL) {
1258 sched(diskp)->act_size = (off_t)0;
1259 allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1260 sched(diskp)->activehd = assign_holdingdisk(holdp, diskp);
1262 sched(diskp)->destname = newstralloc(sched(diskp)->destname,
1263 sched(diskp)->holdp[0]->destname);
1264 diskp->host->inprogress++; /* host is now busy */
1265 diskp->inprogress = 1;
1266 sched(diskp)->dumper = dumper;
1267 sched(diskp)->timestamp = now;
1268 amfree(diskp->dataport_list);
1270 dumper->busy = 1; /* dumper is now busy */
1271 dumper->dp = diskp; /* link disk to dumper */
1272 remove_disk(rq, diskp); /* take it off the run queue */
1274 sched(diskp)->origsize = (off_t)-1;
1275 sched(diskp)->dumpsize = (off_t)-1;
1276 sched(diskp)->dumptime = (time_t)0;
1277 sched(diskp)->tapetime = (time_t)0;
1278 chunker = dumper->chunker = &chktable[dumper - dmptable];
1279 chunker->result = LAST_TOK;
1280 dumper->result = LAST_TOK;
1281 startup_chunk_process(chunker,chunker_program);
1282 chunker_cmd(chunker, START, NULL, driver_timestamp);
1283 chunker->dumper = dumper;
1284 chunker_cmd(chunker, PORT_WRITE, diskp, NULL);
1285 cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
1287 assignedhd_t **h=NULL;
1289 char *qname = quote_string(diskp->name);
1291 g_printf(_("driver: did not get PORT from %s for %s:%s\n"),
1292 chunker->name, diskp->host->hostname, qname);
1296 deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1297 h = sched(diskp)->holdp;
1298 activehd = sched(diskp)->activehd;
1299 h[activehd]->used = 0;
1300 h[activehd]->disk->allocated_dumpers--;
1301 adjust_diskspace(diskp, DONE);
1302 delete_diskspace(diskp);
1303 diskp->host->inprogress--;
1304 diskp->inprogress = 0;
1305 sched(diskp)->dumper = NULL;
1308 sched(diskp)->dump_attempted++;
1309 free_serial_dp(diskp);
1310 if(sched(diskp)->dump_attempted < 2)
1311 enqueue_disk(rq, diskp);
1314 dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD,
1315 handle_dumper_result, dumper);
1316 chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD,
1317 handle_chunker_result, chunker);
1318 dumper->output_port = atoi(result_argv[1]);
1319 amfree(diskp->dataport_list);
1320 diskp->dataport_list = stralloc(result_argv[2]);
1322 if (diskp->host->pre_script == 0) {
1323 run_server_host_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1324 get_config_name(), diskp->host);
1325 diskp->host->pre_script = 1;
1327 run_server_dle_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1328 get_config_name(), diskp,
1329 sched(diskp)->level);
1330 dumper_cmd(dumper, PORT_DUMP, diskp, NULL);
1332 diskp->host->start_t = now + 15;
1333 if (empty(*rq)) force_flush = 1;
1336 g_strfreev(result_argv);
1338 } else if (diskp != NULL && taper != NULL) { /* dump to tape */
1339 sched(diskp)->act_size = (off_t)0;
1340 allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1341 diskp->host->inprogress++; /* host is now busy */
1342 diskp->inprogress = 1;
1343 sched(diskp)->dumper = dumper;
1344 sched(diskp)->taper = taper;
1345 sched(diskp)->timestamp = now;
1346 dumper->chunker = NULL;
1347 amfree(diskp->dataport_list);
1349 dumper->busy = 1; /* dumper is now busy */
1350 dumper->dp = diskp; /* link disk to dumper */
1351 remove_disk(&directq, diskp); /* take it off the direct queue */
1353 sched(diskp)->origsize = (off_t)-1;
1354 sched(diskp)->dumpsize = (off_t)-1;
1355 sched(diskp)->dumptime = (time_t)0;
1356 sched(diskp)->tapetime = (time_t)0;
1357 dumper->result = LAST_TOK;
1358 taper->result = LAST_TOK;
1359 taper->input_error = NULL;
1360 taper->tape_error = NULL;
1361 taper->disk = diskp;
1362 taper->first_label = NULL;
1364 taper->dumper = dumper;
1365 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1366 taper->state &= ~TAPER_STATE_IDLE;
1367 if (taper_nb_wait_reply == 0) {
1368 taper_ev_read = event_register(taper_fd, EV_READFD,
1369 handle_taper_result, NULL);
1372 taper_nb_wait_reply++;
1373 taper_cmd(PORT_WRITE, diskp, NULL, sched(diskp)->level,
1374 sched(diskp)->datestamp);
1375 diskp->host->start_t = now + 15;
1383 * This gets called when a dumper is delayed for some reason. It may
1384 * be because a disk has a delayed start, or amanda is constrained
1385 * by network or disk limits.
1389 handle_dumpers_time(
1392 disklist_t *runq = cookie;
1393 event_release(dumpers_ev_time);
1394 dumpers_ev_time = NULL;
1395 start_some_dumps(runq);
1406 g_printf(_("dump of driver schedule %s:\n--------\n"), str);
1408 for(dp = qp->head; dp != NULL; dp = dp->next) {
1409 qname = quote_string(dp->name);
1410 g_printf(" %-20s %-25s lv %d t %5lu s %lld p %d\n",
1411 dp->host->hostname, qname, sched(dp)->level,
1412 sched(dp)->est_time,
1413 (long long)sched(dp)->est_size, sched(dp)->priority);
1416 g_printf("--------\n");
1420 start_degraded_mode(
1421 /*@keep@*/ disklist_t *queuep)
1425 off_t est_full_size;
1428 newq.head = newq.tail = 0;
1430 dump_schedule(queuep, _("before start degraded mode"));
1432 est_full_size = (off_t)0;
1433 while(!empty(*queuep)) {
1434 dp = dequeue_disk(queuep);
1436 qname = quote_string(dp->name);
1437 if(sched(dp)->level != 0)
1438 /* go ahead and do the disk as-is */
1439 enqueue_disk(&newq, dp);
1441 if (reserved_space + est_full_size + sched(dp)->est_size
1442 <= total_disksize) {
1443 enqueue_disk(&newq, dp);
1444 est_full_size += sched(dp)->est_size;
1446 else if(sched(dp)->degr_level != -1) {
1447 sched(dp)->level = sched(dp)->degr_level;
1448 sched(dp)->dumpdate = sched(dp)->degr_dumpdate;
1449 sched(dp)->est_nsize = sched(dp)->degr_nsize;
1450 sched(dp)->est_csize = sched(dp)->degr_csize;
1451 sched(dp)->est_time = sched(dp)->degr_time;
1452 sched(dp)->est_kps = sched(dp)->degr_kps;
1453 enqueue_disk(&newq, dp);
1456 log_add(L_FAIL, "%s %s %s %d [%s]",
1457 dp->host->hostname, qname, sched(dp)->datestamp,
1458 sched(dp)->level, sched(dp)->degr_mesg);
1464 /*@i@*/ *queuep = newq;
1467 dump_schedule(queuep, _("after start degraded mode"));
1472 continue_port_dumps(void)
1476 int active_dumpers=0, busy_dumpers=0, i;
1479 /* First we try to grant diskspace to some dumps waiting for it. */
1480 for( dp = roomq.head; dp; dp = ndp ) {
1482 /* find last holdingdisk used by this dump */
1483 for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) {
1484 (void)h; /* Quiet lint */
1486 /* find more space */
1487 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1488 &active_dumpers, h[i] );
1490 for(dumper = dmptable; dumper < dmptable + inparallel &&
1491 dumper->dp != dp; dumper++) {
1492 (void)dp; /* Quiet lint */
1494 assert( dumper < dmptable + inparallel );
1495 sched(dp)->activehd = assign_holdingdisk( h, dp );
1496 chunker_cmd( dumper->chunker, CONTINUE, dp, NULL );
1498 remove_disk( &roomq, dp );
1502 /* So for some disks there is less holding diskspace available than
1503 * was asked for. Possible reasons are
1504 * a) diskspace has been allocated for other dumps which are
1505 * still running or already being written to tape
1506 * b) all other dumps have been suspended due to lack of diskspace
1507 * Case a) is not a problem. We just wait for the diskspace to
1508 * be freed by moving the current disk to a queue.
1509 * If case b) occurs, we have a deadlock situation. We select
1510 * a dump from the queue to be aborted and abort it. It will
1511 * be retried directly to tape.
1513 for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1514 if( dumper->busy ) {
1516 if( !find_disk(&roomq, dumper->dp) ) {
1517 if (dumper->chunker) {
1521 sched(dp)->est_size > sched(dumper->dp)->est_size ) {
1526 if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) &&
1527 ((no_taper_flushing() && empty(tapeq)) || degraded_mode) &&
1528 pending_aborts == 0 ) { /* case b */
1529 sched(dp)->no_space = 1;
1530 /* At this time, dp points to the dump with the smallest est_size.
1531 * We abort that dump, hopefully not wasting too much time retrying it.
1533 remove_disk( &roomq, dp );
1534 chunker_cmd(sched(dp)->dumper->chunker, ABORT, NULL, _("Not enough holding disk space"));
1535 dumper_cmd( sched(dp)->dumper, ABORT, NULL, _("Not enough holding disk space"));
1542 handle_taper_result(
1543 void *cookie G_GNUC_UNUSED)
1552 taper_t *taper = NULL;
1557 assert(cookie == NULL);
1564 cmd = getresult(taper_fd, 1, &result_argc, &result_argv);
1569 if(result_argc != 2) {
1570 error(_("error: [taper FAILED result_argc != 2: %d"), result_argc);
1576 for (i=0; i < conf_taper_parallel_write; i++) {
1577 if (strcmp(tapetable[i].name, result_argv[1]) == 0) {
1578 taper= &tapetable[i];
1581 assert(taper != NULL);
1583 taper->state &= ~TAPER_STATE_INIT;
1584 taper->state |= TAPER_STATE_RESERVATION;
1585 taper->state |= TAPER_STATE_IDLE;
1586 amfree(taper->first_label);
1587 taper_nb_wait_reply--;
1588 taper_nb_scan_volume--;
1589 last_started_taper = taper;
1590 if (taper_nb_wait_reply == 0) {
1591 event_release(taper_ev_read);
1592 taper_ev_read = NULL;
1594 start_some_dumps(&runq);
1598 case FAILED: /* FAILED <handle> INPUT-* TAPE-* <input err mesg> <tape err mesg> */
1599 if(result_argc != 6) {
1600 error(_("error: [taper FAILED result_argc != 6: %d"), result_argc);
1604 dp = serial2disk(result_argv[1]);
1605 taper = sched(dp)->taper;
1606 assert(dp == taper->disk);
1608 free_serial(result_argv[1]);
1610 qname = quote_string(dp->name);
1611 g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1612 walltime_str(curclock()), dp->host->hostname, qname);
1615 if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1616 taper->input_error = newstralloc(taper->input_error, result_argv[4]);
1617 taper->result = FAILED;
1620 } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1621 taper->tape_error = newstralloc(taper->tape_error,
1622 _("Taper protocol error"));
1623 taper->result = FAILED;
1624 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1625 dp->host->hostname, qname, sched(dp)->datestamp,
1626 sched(dp)->level, taper->tape_error);
1630 if (strcmp(result_argv[3], "TAPE-ERROR") == 0) {
1631 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1632 taper->tape_error = newstralloc(taper->tape_error, result_argv[5]);
1633 taper->result = FAILED;
1636 } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1637 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1638 taper->tape_error = newstralloc(taper->tape_error,
1639 _("Taper protocol error"));
1640 taper->result = FAILED;
1641 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1642 dp->host->hostname, qname, sched(dp)->datestamp,
1643 sched(dp)->level, taper->tape_error);
1649 taper->result = cmd;
1653 case PARTIAL: /* PARTIAL <handle> INPUT-* TAPE-* <stat mess> <input err mesg> <tape err mesg>*/
1654 case DONE: /* DONE <handle> INPUT-GOOD TAPE-GOOD <stat mess> <input err mesg> <tape err mesg> */
1655 if(result_argc != 7) {
1656 error(_("error: [taper PARTIAL result_argc != 7: %d"), result_argc);
1660 dp = serial2disk(result_argv[1]);
1661 taper = sched(dp)->taper;
1662 assert(dp == taper->disk);
1664 free_serial(result_argv[1]);
1666 qname = quote_string(dp->name);
1667 g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1668 walltime_str(curclock()), dp->host->hostname, qname);
1671 if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1672 taper->input_error = newstralloc(taper->input_error, result_argv[5]);
1673 taper->result = FAILED;
1676 } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1677 taper->tape_error = newstralloc(taper->tape_error,
1678 _("Taper protocol error"));
1679 taper->result = FAILED;
1680 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1681 dp->host->hostname, qname, sched(dp)->datestamp,
1682 sched(dp)->level, taper->tape_error);
1686 if (strcmp(result_argv[3], "TAPE-ERROR") == 0) {
1687 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1688 taper->tape_error = newstralloc(taper->tape_error, result_argv[6]);
1689 taper->result = FAILED;
1692 } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1693 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1694 taper->tape_error = newstralloc(taper->tape_error,
1695 _("Taper protocol error"));
1696 taper->result = FAILED;
1697 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1698 dp->host->hostname, qname, sched(dp)->datestamp,
1699 sched(dp)->level, taper->tape_error);
1704 s = strstr(result_argv[4], " kb ");
1707 sched(dp)->dumpsize = atol(s);
1709 s = strstr(result_argv[4], " bytes ");
1712 sched(dp)->dumpsize = atol(s)/1024;
1716 taper->result = cmd;
1721 case PARTDONE: /* PARTDONE <handle> <label> <fileno> <kbytes> <stat> */
1722 dp = serial2disk(result_argv[1]);
1723 taper = sched(dp)->taper;
1724 assert(dp == taper->disk);
1725 if (result_argc != 6) {
1726 error(_("error [taper PARTDONE result_argc != 6: %d]"),
1730 if (!taper->first_label) {
1731 amfree(taper->first_label);
1732 taper->first_label = stralloc(result_argv[2]);
1733 taper->first_fileno = OFF_T_ATOI(result_argv[3]);
1735 taper->written += OFF_T_ATOI(result_argv[4]);
1736 if (taper->written > sched(taper->disk)->act_size)
1737 sched(taper->disk)->act_size = taper->written;
1740 s = strstr(result_argv[5], " kb ");
1745 s = strstr(result_argv[5], " bytes ");
1748 partsize = atol(s)/1024;
1751 taper->left -= partsize;
1755 case REQUEST_NEW_TAPE: /* REQUEST-NEW-TAPE <handle> */
1756 if (result_argc != 2) {
1757 error(_("error [taper REQUEST_NEW_TAPE result_argc != 2: %d]"),
1762 dp = serial2disk(result_argv[1]);
1763 taper = sched(dp)->taper;
1764 if (taper->state & TAPER_STATE_DONE) {
1765 taper_cmd(NO_NEW_TAPE, taper->disk, "taper found no tape", 0, NULL);
1767 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1768 taper->state |= TAPER_STATE_TAPE_REQUESTED;
1770 start_some_dumps(&runq);
1775 case NEW_TAPE: /* NEW-TAPE <handle> <label> */
1776 if (result_argc != 3) {
1777 error(_("error [taper NEW_TAPE result_argc != 3: %d]"),
1783 taper_nb_scan_volume--;
1784 dp = serial2disk(result_argv[1]);
1785 taper = sched(dp)->taper;
1786 /* Update our tape counter and reset taper->left */
1788 taper->left = tape_length;
1789 taper->state &= ~TAPER_STATE_WAIT_NEW_TAPE;
1790 taper->state |= TAPER_STATE_TAPE_STARTED;
1791 last_started_taper = NULL;
1793 /* start a new worker */
1794 for (i = 0; i < conf_taper_parallel_write ; i++) {
1795 taper1 = &tapetable[i];
1796 if (need_degraded == 0 &&
1797 taper1->state == TAPER_STATE_DEFAULT) {
1798 taper1->state = TAPER_STATE_INIT;
1799 if (taper_nb_wait_reply == 0) {
1800 taper_ev_read = event_register(taper_fd, EV_READFD,
1801 handle_taper_result, NULL);
1803 taper_nb_wait_reply++;
1804 taper_nb_scan_volume++;
1805 taper_cmd(START_TAPER, NULL, taper1->name, 0,
1812 case NO_NEW_TAPE: /* NO-NEW-TAPE <handle> */
1813 if (result_argc != 2) {
1814 error(_("error [taper NO_NEW_TAPE result_argc != 2: %d]"),
1819 taper_nb_scan_volume--;
1820 dp = serial2disk(result_argv[1]);
1821 taper = sched(dp)->taper;
1822 taper->state |= TAPER_STATE_DONE;
1823 last_started_taper = NULL;
1824 start_degraded_mode(&runq);
1827 case DUMPER_STATUS: /* DUMPER-STATUS <handle> */
1828 if (result_argc != 2) {
1829 error(_("error [taper DUMPER_STATUS result_argc != 2: %d]"),
1833 dp = serial2disk(result_argv[1]);
1834 taper = sched(dp)->taper;
1835 if (taper->dumper->result == LAST_TOK) {
1836 taper->sendresult = 1;
1838 if( taper->dumper->result == DONE) {
1839 taper_cmd(DONE, dp, NULL, 0, NULL);
1841 taper_cmd(FAILED, dp, NULL, 0, NULL);
1846 case TAPE_ERROR: /* TAPE-ERROR <name> <err mess> */
1848 if (strcmp(result_argv[1], "SETUP") == 0) {
1849 taper_nb_wait_reply = 0;
1850 taper_nb_scan_volume = 0;
1852 taper = taper_from_name(result_argv[1]);
1853 taper->state = TAPER_STATE_DONE;
1855 q = quote_string(result_argv[2]);
1856 log_add(L_WARNING, _("Taper error: %s"), q);
1859 taper->tape_error = newstralloc(taper->tape_error,
1863 taper_nb_wait_reply--;
1864 taper_nb_scan_volume--;
1866 if (taper_nb_wait_reply == 0) {
1867 event_release(taper_ev_read);
1868 taper_ev_read = NULL;
1871 if (schedule_done && !degraded_mode) {
1872 start_degraded_mode(&runq);
1874 start_some_dumps(&runq);
1877 case PORT: /* PORT <name> <handle> <port> <dataport_list> */
1878 dp = serial2disk(result_argv[2]);
1879 taper = sched(dp)->taper;
1880 dumper = sched(dp)->dumper;
1881 dumper->output_port = atoi(result_argv[3]);
1882 amfree(dp->dataport_list);
1883 dp->dataport_list = stralloc(result_argv[4]);
1885 amfree(taper->input_error);
1886 amfree(taper->tape_error);
1887 amfree(taper->first_label);
1889 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1891 if (dp->host->pre_script == 0) {
1892 run_server_host_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1893 get_config_name(), dp->host);
1894 dp->host->pre_script = 1;
1896 run_server_dle_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1897 get_config_name(), dp,
1899 /* tell the dumper to dump to a port */
1900 dumper_cmd(dumper, PORT_DUMP, dp, NULL);
1901 dp->host->start_t = time(NULL) + 15;
1902 amfree(dp->dataport_list);
1904 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1906 dumper->ev_read = event_register(dumper->fd, EV_READFD,
1907 handle_dumper_result, dumper);
1911 log_add(L_WARNING, _("Taper protocol error"));
1913 * Since we received a taper error, we can't send anything more
1914 * to the taper. Go into degraded mode to try to get everthing
1915 * onto disk. Later, these dumps can be flushed to a new tape.
1916 * The tape queue is zapped so that it appears empty in future
1917 * checks. If there are dumps waiting for diskspace to be freed,
1923 _("going into degraded mode because of taper component error."));
1926 for (taper = tapetable;
1927 taper < tapetable + conf_taper_parallel_write;
1929 if (taper && taper->disk && taper->result != LAST_TOK) {
1930 taper->tape_error = newstralloc(taper->tape_error,"BOGUS");
1931 taper->result = cmd;
1932 if (taper->dumper) {
1933 if (taper->dumper->result != LAST_TOK) {
1934 // Dumper already returned it's result
1935 dumper_taper_result(taper->disk);
1938 file_taper_result(taper->disk);
1945 if(taper_ev_read != NULL) {
1946 event_release(taper_ev_read);
1947 taper_ev_read = NULL;
1948 taper_nb_wait_reply = 0;
1950 start_degraded_mode(&runq);
1951 tapeq.head = tapeq.tail = NULL;
1957 error(_("driver received unexpected token (%s) from taper"),
1962 g_strfreev(result_argv);
1964 if (taper && taper->disk && taper->result != LAST_TOK) {
1966 if (taper->dumper->result != LAST_TOK) {
1967 // Dumper already returned it's result
1968 dumper_taper_result(taper->disk);
1971 file_taper_result(taper->disk);
1975 } while(areads_dataready(taper_fd));
1976 start_some_dumps(&runq);
1986 char *qname = quote_string(dp->name);
1988 taper = sched(dp)->taper;
1989 if (taper->result == DONE) {
1990 update_info_taper(dp, taper->first_label, taper->first_fileno,
1994 sched(dp)->taper_attempted += 1;
1996 if (taper->input_error) {
1997 g_printf("driver: taper failed %s %s: %s\n",
1998 dp->host->hostname, qname, taper->input_error);
1999 if (strcmp(sched(dp)->datestamp, driver_timestamp) == 0) {
2000 if(sched(dp)->taper_attempted >= 2) {
2001 log_add(L_FAIL, _("%s %s %s %d [too many taper retries after holding disk error: %s]"),
2002 dp->host->hostname, qname, sched(dp)->datestamp,
2003 sched(dp)->level, taper->input_error);
2004 g_printf("driver: taper failed %s %s, too many taper retry after holding disk error\n",
2005 dp->host->hostname, qname);
2006 amfree(sched(dp)->destname);
2007 amfree(sched(dp)->dumpdate);
2008 amfree(sched(dp)->degr_dumpdate);
2009 amfree(sched(dp)->degr_mesg);
2010 amfree(sched(dp)->datestamp);
2013 log_add(L_INFO, _("%s %s %s %d [Will retry dump because of holding disk error: %s]"),
2014 dp->host->hostname, qname, sched(dp)->datestamp,
2015 sched(dp)->level, taper->input_error);
2016 g_printf("driver: taper will retry %s %s because of holding disk error\n",
2017 dp->host->hostname, qname);
2018 if (dp->to_holdingdisk != HOLD_REQUIRED) {
2019 dp->to_holdingdisk = HOLD_NEVER;
2020 sched(dp)->dump_attempted -= 1;
2021 headqueue_disk(&directq, dp);
2023 amfree(sched(dp)->destname);
2024 amfree(sched(dp)->dumpdate);
2025 amfree(sched(dp)->degr_dumpdate);
2026 amfree(sched(dp)->degr_mesg);
2027 amfree(sched(dp)->datestamp);
2032 amfree(sched(dp)->destname);
2033 amfree(sched(dp)->dumpdate);
2034 amfree(sched(dp)->degr_dumpdate);
2035 amfree(sched(dp)->degr_mesg);
2036 amfree(sched(dp)->datestamp);
2039 } else if (taper->tape_error) {
2040 g_printf("driver: taper failed %s %s with tape error: %s\n",
2041 dp->host->hostname, qname, taper->tape_error);
2042 if(sched(dp)->taper_attempted >= 2) {
2043 log_add(L_FAIL, _("%s %s %s %d [too many taper retries]"),
2044 dp->host->hostname, qname, sched(dp)->datestamp,
2046 g_printf("driver: taper failed %s %s, too many taper retry\n",
2047 dp->host->hostname, qname);
2048 amfree(sched(dp)->destname);
2049 amfree(sched(dp)->dumpdate);
2050 amfree(sched(dp)->degr_dumpdate);
2051 amfree(sched(dp)->degr_mesg);
2052 amfree(sched(dp)->datestamp);
2055 g_printf("driver: taper will retry %s %s\n",
2056 dp->host->hostname, qname);
2057 /* Re-insert into taper queue. */
2058 headqueue_disk(&tapeq, dp);
2060 } else if (taper->result != DONE) {
2061 g_printf("driver: taper failed %s %s without error\n",
2062 dp->host->hostname, qname);
2064 delete_diskspace(dp);
2065 amfree(sched(dp)->destname);
2066 amfree(sched(dp)->dumpdate);
2067 amfree(sched(dp)->degr_dumpdate);
2068 amfree(sched(dp)->degr_mesg);
2069 amfree(sched(dp)->datestamp);
2075 taper->state &= ~TAPER_STATE_FILE_TO_TAPE;
2076 taper->state |= TAPER_STATE_IDLE;
2077 amfree(taper->input_error);
2078 amfree(taper->tape_error);
2080 taper_nb_wait_reply--;
2081 if (taper_nb_wait_reply == 0) {
2082 event_release(taper_ev_read);
2083 taper_ev_read = NULL;
2086 /* continue with those dumps waiting for diskspace */
2087 continue_port_dumps();
2088 start_some_dumps(&runq);
2093 dumper_taper_result(
2101 dumper = sched(dp)->dumper;
2102 taper = sched(dp)->taper;
2105 if(dumper->result == DONE && taper->result == DONE) {
2106 update_info_dumper(dp, sched(dp)->origsize,
2107 sched(dp)->dumpsize, sched(dp)->dumptime);
2108 update_info_taper(dp, taper->first_label, taper->first_fileno,
2110 qname = quote_string(dp->name); /*quote to take care of spaces*/
2112 log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2113 dp->host->hostname, qname, sched(dp)->datestamp,
2115 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2116 (long long)sched(dp)->est_csize,
2117 sched(dp)->est_kps);
2120 update_failed_dump(dp);
2123 is_partial = dumper->result != DONE || taper->result != DONE;
2125 sched(dp)->dump_attempted += 1;
2126 sched(dp)->taper_attempted += 1;
2128 if((dumper->result != DONE || taper->result != DONE) &&
2129 sched(dp)->dump_attempted <= 1 &&
2130 sched(dp)->taper_attempted <= 1) {
2131 enqueue_disk(&directq, dp);
2134 if(dumper->ev_read != NULL) {
2135 event_release(dumper->ev_read);
2136 dumper->ev_read = NULL;
2138 taper_nb_wait_reply--;
2139 if (taper_nb_wait_reply == 0 && taper_ev_read != NULL) {
2140 event_release(taper_ev_read);
2141 taper_ev_read = NULL;
2143 taper->state &= ~TAPER_STATE_DUMP_TO_TAPE;
2144 taper->state |= TAPER_STATE_IDLE;
2145 amfree(taper->input_error);
2146 amfree(taper->tape_error);
2148 dp->host->inprogress -= 1;
2150 deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2151 taper->dumper = NULL;
2153 sched(dp)->dumper = NULL;
2154 sched(dp)->taper = NULL;
2155 start_some_dumps(&runq);
2164 /* Use an already started taper first */
2165 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2167 if ((taper->state & TAPER_STATE_IDLE) &&
2168 (taper->state & TAPER_STATE_TAPE_STARTED) &&
2169 !(taper->state & TAPER_STATE_DONE) &&
2170 !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2171 !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2174 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2176 if ((taper->state & TAPER_STATE_IDLE) &&
2177 (taper->state & TAPER_STATE_RESERVATION) &&
2178 !(taper->state & TAPER_STATE_DONE) &&
2179 !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2180 !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2192 for (taper = tapetable; taper < tapetable+conf_taper_parallel_write;
2194 if (strcmp(taper->name, name) == 0) return taper;
2200 dumper_chunker_result(
2205 assignedhd_t **h=NULL;
2212 dumper = sched(dp)->dumper;
2213 chunker = dumper->chunker;
2217 h = sched(dp)->holdp;
2218 activehd = sched(dp)->activehd;
2220 if(dumper->result == DONE && chunker->result == DONE) {
2221 update_info_dumper(dp, sched(dp)->origsize,
2222 sched(dp)->dumpsize, sched(dp)->dumptime);
2223 qname = quote_string(dp->name);/*quote to take care of spaces*/
2225 log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2226 dp->host->hostname, qname, sched(dp)->datestamp,
2228 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2229 (long long)sched(dp)->est_csize,
2230 sched(dp)->est_kps);
2233 update_failed_dump(dp);
2236 deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2238 is_partial = dumper->result != DONE || chunker->result != DONE;
2239 rename_tmp_holding(sched(dp)->destname, !is_partial);
2240 holding_set_origsize(sched(dp)->destname, sched(dp)->origsize);
2243 for( i = 0, h = sched(dp)->holdp; i < activehd; i++ ) {
2244 dummy += h[i]->used;
2247 size = holding_file_size(sched(dp)->destname, 0);
2248 h[activehd]->used = size - dummy;
2249 h[activehd]->disk->allocated_dumpers--;
2250 adjust_diskspace(dp, DONE);
2252 sched(dp)->dump_attempted += 1;
2254 if((dumper->result != DONE || chunker->result != DONE) &&
2255 sched(dp)->dump_attempted <= 1) {
2256 delete_diskspace(dp);
2257 if (sched(dp)->no_space) {
2258 enqueue_disk(&directq, dp);
2260 enqueue_disk(&runq, dp);
2263 else if(size > (off_t)DISK_BLOCK_KB) {
2264 enqueue_disk(&tapeq, dp);
2267 delete_diskspace(dp);
2271 dp->host->inprogress -= 1;
2274 waitpid(chunker->pid, NULL, 0 );
2275 aclose(chunker->fd);
2280 if (chunker->result == ABORT_FINISHED)
2282 continue_port_dumps();
2284 * Wakeup any dumpers that are sleeping because of network
2285 * or disk constraints.
2287 start_some_dumps(&runq);
2293 handle_dumper_result(
2296 /* uses global pending_aborts */
2297 dumper_t *dumper = cookie;
2299 disk_t *dp, *sdp, *dp1;
2305 assert(dumper != NULL);
2308 assert(sched(dp) != NULL);
2313 cmd = getresult(dumper->fd, 1, &result_argc, &result_argv);
2316 /* result_argv[1] always contains the serial number */
2317 sdp = serial2disk(result_argv[1]);
2319 error(_("Invalid serial number %s"), result_argv[1]);
2320 g_assert_not_reached();
2324 qname = quote_string(dp->name);
2327 case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
2328 if(result_argc != 6) {
2329 error(_("error [dumper DONE result_argc != 6: %d]"), result_argc);
2333 sched(dp)->origsize = OFF_T_ATOI(result_argv[2]);
2334 sched(dp)->dumptime = TIME_T_ATOI(result_argv[4]);
2336 g_printf(_("driver: finished-cmd time %s %s dumped %s:%s\n"),
2337 walltime_str(curclock()), dumper->name,
2338 dp->host->hostname, qname);
2341 dumper->result = cmd;
2345 case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2347 * Requeue this disk, and fall through to the FAILED
2350 if(sched(dp)->dump_attempted) {
2351 char *qname = quote_string(dp->name);
2352 char *qerr = quote_string(result_argv[2]);
2353 log_add(L_FAIL, _("%s %s %s %d [too many dumper retry: %s]"),
2354 dp->host->hostname, qname, sched(dp)->datestamp,
2355 sched(dp)->level, qerr);
2356 g_printf(_("driver: dump failed %s %s %s, too many dumper retry: %s\n"),
2357 result_argv[1], dp->host->hostname, qname, qerr);
2362 case FAILED: /* FAILED <handle> <errstr> */
2363 /*free_serial(result_argv[1]);*/
2364 dumper->result = cmd;
2367 case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2369 * We sent an ABORT from the NO-ROOM case because this dump
2370 * wasn't going to fit onto the holding disk. We now need to
2371 * clean up the remains of this image, and try to finish
2372 * other dumps that are waiting on disk space.
2374 assert(pending_aborts);
2375 /*free_serial(result_argv[1]);*/
2376 dumper->result = cmd;
2380 /* either EOF or garbage from dumper. Turn it off */
2381 log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2382 dumper->name, (long)dumper->pid);
2383 if (dumper->ev_read) {
2384 event_release(dumper->ev_read);
2385 dumper->ev_read = NULL;
2389 dumper->down = 1; /* mark it down so it isn't used again */
2391 /* if it was dumping something, zap it and try again */
2392 if(sched(dp)->dump_attempted) {
2393 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2394 dp->host->hostname, qname, sched(dp)->datestamp,
2395 sched(dp)->level, dumper->name);
2397 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2398 dumper->name, dp->host->hostname, qname,
2401 dumper->result = cmd;
2408 g_strfreev(result_argv);
2414 run_server_dle_scripts(EXECUTE_ON_POST_DLE_BACKUP,
2415 get_config_name(), dp, sched(dp)->level);
2416 /* check dump not yet started */
2417 for (dp1=runq.head; dp1 != NULL; dp1 = dp1->next) {
2418 if (dp1->host == dp->host)
2421 /* check direct to tape dump */
2422 for (dp1=directq.head; dp1 != NULL; dp1 = dp1->next) {
2423 if (dp1->host == dp->host)
2426 /* check dumping dle */
2427 for (dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
2428 if (dumper->busy && dumper->dp != dp &&
2429 dumper->dp->host == dp->host)
2432 if (last_dump && dp->host->post_script == 0) {
2433 if (dp->host->post_script == 0) {
2434 run_server_host_scripts(EXECUTE_ON_POST_HOST_BACKUP,
2435 get_config_name(), dp->host);
2436 dp->host->post_script = 1;
2441 taper = sched(dp)->taper;
2442 /* send the dumper result to the chunker */
2443 if (dumper->chunker) {
2444 if (dumper->chunker->down == 0 && dumper->chunker->fd != -1 &&
2445 dumper->chunker->result == LAST_TOK) {
2447 chunker_cmd(dumper->chunker, DONE, dp, NULL);
2450 chunker_cmd(dumper->chunker, FAILED, dp, NULL);
2453 if( dumper->result != LAST_TOK &&
2454 dumper->chunker->result != LAST_TOK)
2455 dumper_chunker_result(dp);
2456 } else { /* send the dumper result to the taper */
2457 if (taper->sendresult) {
2459 taper_cmd(DONE, dp, NULL, 0, NULL);
2461 taper_cmd(FAILED, dp, NULL, 0, NULL);
2463 taper->sendresult = 0;
2465 if (taper->dumper && taper->result != LAST_TOK) {
2466 dumper_taper_result(dp);
2469 } while(areads_dataready(dumper->fd));
2474 handle_chunker_result(
2477 chunker_t *chunker = cookie;
2478 assignedhd_t **h=NULL;
2488 assert(chunker != NULL);
2489 dumper = chunker->dumper;
2490 assert(dumper != NULL);
2493 assert(sched(dp) != NULL);
2494 assert(sched(dp)->destname != NULL);
2495 assert(dp != NULL && sched(dp) != NULL && sched(dp)->destname);
2497 if(sched(dp)->holdp) {
2498 h = sched(dp)->holdp;
2499 activehd = sched(dp)->activehd;
2505 cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
2508 /* result_argv[1] always contains the serial number */
2509 sdp = serial2disk(result_argv[1]);
2511 error(_("Invalid serial number %s"), result_argv[1]);
2512 g_assert_not_reached();
2518 case PARTIAL: /* PARTIAL <handle> <dumpsize> <errstr> */
2519 case DONE: /* DONE <handle> <dumpsize> <errstr> */
2520 if(result_argc != 4) {
2521 error(_("error [chunker %s result_argc != 4: %d]"), cmdstr[cmd],
2525 /*free_serial(result_argv[1]);*/
2527 sched(dp)->dumpsize = (off_t)atof(result_argv[2]);
2529 qname = quote_string(dp->name);
2530 g_printf(_("driver: finished-cmd time %s %s chunked %s:%s\n"),
2531 walltime_str(curclock()), chunker->name,
2532 dp->host->hostname, qname);
2536 event_release(chunker->ev_read);
2538 chunker->result = cmd;
2542 case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2543 event_release(chunker->ev_read);
2545 chunker->result = cmd;
2548 case FAILED: /* FAILED <handle> <errstr> */
2549 /*free_serial(result_argv[1]);*/
2551 event_release(chunker->ev_read);
2553 chunker->result = cmd;
2557 case NO_ROOM: /* NO-ROOM <handle> <missing_size> */
2558 if (!h || activehd < 0) { /* should never happen */
2559 error(_("!h || activehd < 0"));
2562 h[activehd]->used -= OFF_T_ATOI(result_argv[2]);
2563 h[activehd]->reserved -= OFF_T_ATOI(result_argv[2]);
2564 h[activehd]->disk->allocated_space -= OFF_T_ATOI(result_argv[2]);
2565 h[activehd]->disk->disksize -= OFF_T_ATOI(result_argv[2]);
2568 case RQ_MORE_DISK: /* RQ-MORE-DISK <handle> */
2569 if (!h || activehd < 0) { /* should never happen */
2570 error(_("!h || activehd < 0"));
2573 h[activehd]->disk->allocated_dumpers--;
2574 h[activehd]->used = h[activehd]->reserved;
2575 if( h[++activehd] ) { /* There's still some allocated space left.
2576 * Tell the dumper about it. */
2577 sched(dp)->activehd++;
2578 chunker_cmd( chunker, CONTINUE, dp, NULL );
2579 } else { /* !h[++activehd] - must allocate more space */
2580 sched(dp)->act_size = sched(dp)->est_size; /* not quite true */
2581 sched(dp)->est_size = (sched(dp)->act_size/(off_t)20) * (off_t)21; /* +5% */
2582 sched(dp)->est_size = am_round(sched(dp)->est_size, (off_t)DISK_BLOCK_KB);
2583 if (sched(dp)->est_size < sched(dp)->act_size + 2*DISK_BLOCK_KB)
2584 sched(dp)->est_size += 2 * DISK_BLOCK_KB;
2585 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
2589 /* No diskspace available. The reason for this will be
2590 * determined in continue_port_dumps(). */
2591 enqueue_disk( &roomq, dp );
2592 continue_port_dumps();
2593 /* continue flush waiting for new tape */
2596 /* OK, allocate space for disk and have chunker continue */
2597 sched(dp)->activehd = assign_holdingdisk( h, dp );
2598 chunker_cmd( chunker, CONTINUE, dp, NULL );
2604 case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2606 * We sent an ABORT from the NO-ROOM case because this dump
2607 * wasn't going to fit onto the holding disk. We now need to
2608 * clean up the remains of this image, and try to finish
2609 * other dumps that are waiting on disk space.
2611 /*assert(pending_aborts);*/
2613 /*free_serial(result_argv[1]);*/
2615 event_release(chunker->ev_read);
2617 chunker->result = cmd;
2622 /* either EOF or garbage from chunker. Turn it off */
2623 log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2624 chunker->name, (long)chunker->pid);
2626 /* if it was dumping something, zap it and try again */
2627 g_assert(h && activehd >= 0);
2628 qname = quote_string(dp->name);
2629 if(sched(dp)->dump_attempted) {
2630 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2631 dp->host->hostname, qname, sched(dp)->datestamp,
2632 sched(dp)->level, chunker->name);
2634 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2635 chunker->name, dp->host->hostname, qname,
2641 event_release(chunker->ev_read);
2643 chunker->result = cmd;
2650 g_strfreev(result_argv);
2652 if(chunker->result != LAST_TOK && chunker->dumper->result != LAST_TOK)
2653 dumper_chunker_result(dp);
2655 } while(areads_dataready(chunker->fd));
2666 char *hostname, *diskname, *datestamp;
2670 char *inpline = NULL;
2675 char *qdestname = NULL;
2676 char *conf_infofile;
2678 (void)cookie; /* Quiet unused parameter warning */
2680 event_release(flush_ev_read);
2681 flush_ev_read = NULL;
2683 /* read schedule from stdin */
2684 conf_infofile = config_dir_relative(getconf_str(CNF_INFOFILE));
2685 if (open_infofile(conf_infofile)) {
2686 error(_("could not open info db \"%s\""), conf_infofile);
2689 amfree(conf_infofile);
2691 for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2695 if (inpline[0] == '\0')
2701 skip_whitespace(s, ch); /* find the command */
2703 error(_("flush line %d: syntax error (no command)"), line);
2707 skip_non_whitespace(s, ch);
2710 if(strcmp(command,"ENDFLUSH") == 0) {
2714 if(strcmp(command,"FLUSH") != 0) {
2715 error(_("flush line %d: syntax error (%s != FLUSH)"), line, command);
2719 skip_whitespace(s, ch); /* find the hostname */
2721 error(_("flush line %d: syntax error (no hostname)"), line);
2725 skip_non_whitespace(s, ch);
2728 skip_whitespace(s, ch); /* find the diskname */
2730 error(_("flush line %d: syntax error (no diskname)"), line);
2734 skip_quoted_string(s, ch);
2735 s[-1] = '\0'; /* terminate the disk name */
2736 diskname = unquote_string(qname);
2738 skip_whitespace(s, ch); /* find the datestamp */
2740 error(_("flush line %d: syntax error (no datestamp)"), line);
2744 skip_non_whitespace(s, ch);
2747 skip_whitespace(s, ch); /* find the level number */
2748 if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2749 error(_("flush line %d: syntax error (bad level)"), line);
2752 skip_integer(s, ch);
2754 skip_whitespace(s, ch); /* find the filename */
2756 error(_("flush line %d: syntax error (no filename)"), line);
2760 skip_quoted_string(s, ch);
2762 destname = unquote_string(qdestname);
2764 holding_file_get_dumpfile(destname, &file);
2765 if( file.type != F_DUMPFILE) {
2766 if( file.type != F_CONT_DUMPFILE )
2767 log_add(L_INFO, _("%s: ignoring cruft file."), destname);
2770 dumpfile_free_data(&file);
2774 if(strcmp(hostname, file.name) != 0 ||
2775 strcmp(diskname, file.disk) != 0 ||
2776 strcmp(datestamp, file.datestamp) != 0) {
2777 log_add(L_INFO, _("disk %s:%s not consistent with file %s"),
2778 hostname, diskname, destname);
2781 dumpfile_free_data(&file);
2786 dp = lookup_disk(file.name, file.disk);
2789 log_add(L_INFO, _("%s: disk %s:%s not in database, skipping it."),
2790 destname, file.name, file.disk);
2792 dumpfile_free_data(&file);
2796 if (file.dumplevel < 0 || file.dumplevel > 399) {
2797 log_add(L_INFO, _("%s: ignoring file with bogus dump level %d."),
2798 destname, file.dumplevel);
2800 dumpfile_free_data(&file);
2804 if (holding_file_size(destname,1) <= 0) {
2805 log_add(L_INFO, "%s: removing file with no data.", destname);
2806 holding_file_unlink(destname);
2808 dumpfile_free_data(&file);
2812 dp1 = (disk_t *)alloc(SIZEOF(disk_t));
2814 dp1->next = dp1->prev = NULL;
2816 /* add it to the flushhost list */
2818 flushhost = alloc(SIZEOF(am_host_t));
2819 flushhost->next = NULL;
2820 flushhost->hostname = stralloc("FLUSHHOST");
2821 flushhost->up = NULL;
2822 flushhost->features = NULL;
2824 dp1->hostnext = flushhost->disks;
2825 flushhost->disks = dp1;
2827 sp = (sched_t *) alloc(SIZEOF(sched_t));
2828 sp->destname = destname;
2829 sp->level = file.dumplevel;
2830 sp->dumpdate = NULL;
2831 sp->degr_dumpdate = NULL;
2832 sp->degr_mesg = NULL;
2833 sp->datestamp = stralloc(file.datestamp);
2834 sp->est_nsize = (off_t)0;
2835 sp->est_csize = (off_t)0;
2838 sp->origsize = file.orig_size;
2840 sp->degr_level = -1;
2841 sp->dump_attempted = 0;
2842 sp->taper_attempted = 0;
2843 sp->act_size = holding_file_size(destname, 0);
2844 sp->holdp = build_diskspace(destname);
2845 if(sp->holdp == NULL) continue;
2848 sp->timestamp = (time_t)0;
2850 dp1->up = (char *)sp;
2852 enqueue_disk(&tapeq, dp1);
2853 dumpfile_free_data(&file);
2860 schedule_ev_read = event_register((event_id_t)0, EV_READFD,
2861 read_schedule, NULL);
2873 int level, line, priority;
2874 char *dumpdate, *degr_dumpdate, *degr_mesg;
2876 time_t time, degr_time;
2877 time_t *time_p = &time;
2878 time_t *degr_time_p = °r_time;
2879 off_t nsize, csize, degr_nsize, degr_csize;
2880 unsigned long kps, degr_kps;
2881 char *hostname, *features, *diskname, *datestamp, *inpline = NULL;
2885 off_t flush_size = (off_t)0;
2890 long long degr_nsize_;
2891 long long degr_csize_;
2892 GPtrArray *errarray;
2894 (void)cookie; /* Quiet unused parameter warning */
2896 event_release(schedule_ev_read);
2897 schedule_ev_read = NULL;
2899 /* read schedule from stdin */
2901 for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2902 if (inpline[0] == '\0')
2909 skip_whitespace(s, ch); /* find the command */
2911 error(_("schedule line %d: syntax error (no command)"), line);
2915 skip_non_whitespace(s, ch);
2918 if(strcmp(command,"DUMP") != 0) {
2919 error(_("schedule line %d: syntax error (%s != DUMP)"), line, command);
2923 skip_whitespace(s, ch); /* find the host name */
2925 error(_("schedule line %d: syntax error (no host name)"), line);
2929 skip_non_whitespace(s, ch);
2932 skip_whitespace(s, ch); /* find the feature list */
2934 error(_("schedule line %d: syntax error (no feature list)"), line);
2938 skip_non_whitespace(s, ch);
2941 skip_whitespace(s, ch); /* find the disk name */
2943 error(_("schedule line %d: syntax error (no disk name)"), line);
2947 skip_quoted_string(s, ch);
2948 s[-1] = '\0'; /* terminate the disk name */
2949 diskname = unquote_string(qname);
2951 skip_whitespace(s, ch); /* find the datestamp */
2953 error(_("schedule line %d: syntax error (no datestamp)"), line);
2957 skip_non_whitespace(s, ch);
2960 skip_whitespace(s, ch); /* find the priority number */
2961 if(ch == '\0' || sscanf(s - 1, "%d", &priority) != 1) {
2962 error(_("schedule line %d: syntax error (bad priority)"), line);
2965 skip_integer(s, ch);
2967 skip_whitespace(s, ch); /* find the level number */
2968 if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2969 error(_("schedule line %d: syntax error (bad level)"), line);
2972 skip_integer(s, ch);
2974 skip_whitespace(s, ch); /* find the dump date */
2976 error(_("schedule line %d: syntax error (bad dump date)"), line);
2980 skip_non_whitespace(s, ch);
2983 skip_whitespace(s, ch); /* find the native size */
2985 if(ch == '\0' || sscanf(s - 1, "%lld", &nsize_) != 1) {
2986 error(_("schedule line %d: syntax error (bad nsize)"), line);
2989 nsize = (off_t)nsize_;
2990 skip_integer(s, ch);
2992 skip_whitespace(s, ch); /* find the compressed size */
2994 if(ch == '\0' || sscanf(s - 1, "%lld", &csize_) != 1) {
2995 error(_("schedule line %d: syntax error (bad csize)"), line);
2998 csize = (off_t)csize_;
2999 skip_integer(s, ch);
3001 skip_whitespace(s, ch); /* find the time number */
3002 if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
3003 error(_("schedule line %d: syntax error (bad estimated time)"), line);
3006 *time_p = (time_t)time_;
3007 skip_integer(s, ch);
3009 skip_whitespace(s, ch); /* find the kps number */
3010 if(ch == '\0' || sscanf(s - 1, "%lu", &kps) != 1) {
3011 error(_("schedule line %d: syntax error (bad kps)"), line);
3014 skip_integer(s, ch);
3016 degr_dumpdate = NULL; /* flag if degr fields found */
3017 skip_whitespace(s, ch); /* find the degr level number */
3021 skip_quoted_string(s, ch);
3022 s[-1] = '\0'; /* terminate degr mesg */
3023 degr_mesg = unquote_string(qname);
3025 degr_nsize = (off_t)0;
3026 degr_csize = (off_t)0;
3027 degr_time = (time_t)0;
3029 } else if (ch != '\0') {
3030 if(sscanf(s - 1, "%d", °r_level) != 1) {
3031 error(_("schedule line %d: syntax error (bad degr level)"), line);
3034 skip_integer(s, ch);
3036 skip_whitespace(s, ch); /* find the degr dump date */
3038 error(_("schedule line %d: syntax error (bad degr dump date)"), line);
3041 degr_dumpdate = s - 1;
3042 skip_non_whitespace(s, ch);
3045 skip_whitespace(s, ch); /* find the degr native size */
3046 degr_nsize_ = (off_t)0;
3047 if(ch == '\0' || sscanf(s - 1, "%lld", °r_nsize_) != 1) {
3048 error(_("schedule line %d: syntax error (bad degr nsize)"), line);
3051 degr_nsize = (off_t)degr_nsize_;
3052 skip_integer(s, ch);
3054 skip_whitespace(s, ch); /* find the degr compressed size */
3055 degr_csize_ = (off_t)0;
3056 if(ch == '\0' || sscanf(s - 1, "%lld", °r_csize_) != 1) {
3057 error(_("schedule line %d: syntax error (bad degr csize)"), line);
3060 degr_csize = (off_t)degr_csize_;
3061 skip_integer(s, ch);
3063 skip_whitespace(s, ch); /* find the degr time number */
3064 if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
3065 error(_("schedule line %d: syntax error (bad degr estimated time)"), line);
3068 *degr_time_p = (time_t)time_;
3069 skip_integer(s, ch);
3071 skip_whitespace(s, ch); /* find the degr kps number */
3072 if(ch == '\0' || sscanf(s - 1, "%lu", °r_kps) != 1) {
3073 error(_("schedule line %d: syntax error (bad degr kps)"), line);
3076 skip_integer(s, ch);
3078 error(_("schedule line %d: no degraded estimate or message"), line);
3081 dp = lookup_disk(hostname, diskname);
3084 _("schedule line %d: %s:'%s' not in disklist, ignored"),
3085 line, hostname, qname);
3090 sp = (sched_t *) alloc(SIZEOF(sched_t));
3093 sp->dumpdate = stralloc(dumpdate);
3094 sp->est_nsize = DISK_BLOCK_KB + nsize; /* include header */
3095 sp->est_csize = DISK_BLOCK_KB + csize; /* include header */
3096 /* round estimate to next multiple of DISK_BLOCK_KB */
3097 sp->est_csize = am_round(sp->est_csize, DISK_BLOCK_KB);
3098 sp->est_size = sp->est_csize;
3099 sp->est_time = time;
3101 sp->priority = priority;
3102 sp->datestamp = stralloc(datestamp);
3105 sp->degr_level = degr_level;
3106 sp->degr_dumpdate = stralloc(degr_dumpdate);
3107 sp->degr_nsize = DISK_BLOCK_KB + degr_nsize;
3108 sp->degr_csize = DISK_BLOCK_KB + degr_csize;
3109 /* round estimate to next multiple of DISK_BLOCK_KB */
3110 sp->degr_csize = am_round(sp->degr_csize, DISK_BLOCK_KB);
3111 sp->degr_time = degr_time;
3112 sp->degr_kps = degr_kps;
3113 sp->degr_mesg = NULL;
3115 sp->degr_level = -1;
3116 sp->degr_dumpdate = NULL;
3117 sp->degr_mesg = degr_mesg;
3121 sp->dump_attempted = 0;
3122 sp->taper_attempted = 0;
3128 sp->timestamp = (time_t)0;
3129 sp->destname = NULL;
3132 dp->up = (char *) sp;
3133 if(dp->host->features == NULL) {
3134 dp->host->features = am_string_to_feature(features);
3135 if (!dp->host->features) {
3137 _("Invalid feature string from client '%s'"),
3139 dp->host->features = am_set_default_feature_set();
3142 remove_disk(&waitq, dp);
3144 errarray = validate_optionstr(dp);
3145 if (errarray->len > 0) {
3147 for (i=0; i < errarray->len; i++) {
3148 log_add(L_FAIL, _("%s %s %s 0 [%s]"),
3149 dp->host->hostname, qname,
3151 (char *)g_ptr_array_index(errarray, i));
3156 if (dp->data_path == DATA_PATH_DIRECTTCP &&
3157 dp->to_holdingdisk == HOLD_AUTO) {
3158 dp->to_holdingdisk = HOLD_NEVER;
3161 if (dp->to_holdingdisk == HOLD_NEVER) {
3162 enqueue_disk(&directq, dp);
3164 enqueue_disk(&runq, dp);
3166 flush_size += sp->act_size;
3170 g_printf(_("driver: flush size %lld\n"), (long long)flush_size);
3173 log_add(L_WARNING, _("WARNING: got empty schedule from planner"));
3174 if(need_degraded==1) start_degraded_mode(&runq);
3176 run_server_global_scripts(EXECUTE_ON_PRE_BACKUP, get_config_name());
3177 if (empty(runq)) force_flush = 1;
3178 start_some_dumps(&runq);
3182 static unsigned long
3190 unsigned long maxusage=0;
3191 unsigned long curusage=0;
3192 for(p = disklist_netifs(); p != NULL; p = p->next) {
3193 maxusage += interface_get_maxusage(p->config);
3194 curusage += p->curusage;
3196 if (maxusage >= curusage)
3197 res = maxusage - curusage;
3202 if ((unsigned long)interface_get_maxusage(ip->config) >= ip->curusage)
3203 res = interface_get_maxusage(ip->config) - ip->curusage;
3218 g_printf(_("driver: interface-state time %s"), time_str);
3220 for(ip = disklist_netifs(); ip != NULL; ip = ip->next) {
3221 g_printf(_(" if %s: free %lu"), interface_name(ip->config), free_kps(ip));
3231 ip->curusage += kps;
3235 deallocate_bandwidth(
3239 assert(kps <= ip->curusage);
3240 ip->curusage -= kps;
3251 total_free = (off_t)0;
3252 for(ha = holdalloc; ha != NULL; ha = ha->next) {
3253 diff = ha->disksize - ha->allocated_space;
3261 * We return an array of pointers to assignedhd_t. The array contains at
3262 * most one entry per holding disk. The list of pointers is terminated by
3263 * a NULL pointer. Each entry contains a pointer to a holdingdisk and
3264 * how much diskspace to use on that disk. Later on, assign_holdingdisk
3265 * will allocate the given amount of space.
3266 * If there is not enough room on the holdingdisks, NULL is returned.
3269 static assignedhd_t **
3273 assignedhd_t * pref)
3275 assignedhd_t **result = NULL;
3276 holdalloc_t *ha, *minp;
3280 off_t halloc, dalloc, hfree, dfree;
3282 (void)cur_idle; /* Quiet unused parameter warning */
3284 if (size < 2*DISK_BLOCK_KB)
3285 size = 2*DISK_BLOCK_KB;
3286 size = am_round(size, (off_t)DISK_BLOCK_KB);
3288 hold_debug(1, _("find_diskspace: want %lld K\n"),
3291 used = alloc(SIZEOF(*used) * num_holdalloc);/*disks used during this run*/
3292 memset( used, 0, (size_t)num_holdalloc );
3293 result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3296 while( i < num_holdalloc && size > (off_t)0 ) {
3297 /* find the holdingdisk with the fewest active dumpers and among
3298 * those the one with the biggest free space
3300 minp = NULL; minj = -1;
3301 for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3302 if( pref && pref->disk == ha && !used[j] &&
3303 ha->allocated_space <= ha->disksize - (off_t)DISK_BLOCK_KB) {
3308 else if( ha->allocated_space <= ha->disksize - (off_t)(2*DISK_BLOCK_KB) &&
3311 ha->allocated_dumpers < minp->allocated_dumpers ||
3312 (ha->allocated_dumpers == minp->allocated_dumpers &&
3313 ha->disksize-ha->allocated_space > minp->disksize-minp->allocated_space)) ) {
3320 if( !minp ) { break; } /* all holding disks are full */
3323 /* hfree = free space on the disk */
3324 hfree = minp->disksize - minp->allocated_space;
3326 /* dfree = free space for data, remove 1 header for each chunksize */
3327 dfree = hfree - (((hfree-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3329 /* dalloc = space I can allocate for data */
3330 dalloc = ( dfree < size ) ? dfree : size;
3332 /* halloc = space to allocate, including 1 header for each chunksize */
3333 halloc = dalloc + (((dalloc-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3335 hold_debug(1, _("find_diskspace: find diskspace: size %lld hf %lld df %lld da %lld ha %lld\n"),
3342 result[i] = alloc(SIZEOF(assignedhd_t));
3343 result[i]->disk = minp;
3344 result[i]->reserved = halloc;
3345 result[i]->used = (off_t)0;
3346 result[i]->destname = NULL;
3352 if(size != (off_t)0) { /* not enough space available */
3353 g_printf(_("find diskspace: not enough diskspace. Left with %lld K\n"), (long long)size);
3355 free_assignedhd(result);
3359 if (debug_holding > 1) {
3360 for( i = 0; result && result[i]; i++ ) {
3361 hold_debug(1, _("find_diskspace: find diskspace: selected %s free %lld reserved %lld dumpers %d\n"),
3362 holdingdisk_get_diskdir(result[i]->disk->hdisk),
3363 (long long)(result[i]->disk->disksize -
3364 result[i]->disk->allocated_space),
3365 (long long)result[i]->reserved,
3366 result[i]->disk->allocated_dumpers);
3375 assignedhd_t ** holdp,
3380 char *sfn = sanitise_filename(diskp->name);
3382 assignedhd_t **new_holdp;
3385 g_snprintf( lvl, SIZEOF(lvl), "%d", sched(diskp)->level );
3387 size = am_round(sched(diskp)->est_size - sched(diskp)->act_size,
3388 (off_t)DISK_BLOCK_KB);
3390 for( c = 0; holdp[c]; c++ )
3391 (void)c; /* count number of disks */
3393 /* allocate memory for sched(diskp)->holdp */
3394 for(j = 0; sched(diskp)->holdp && sched(diskp)->holdp[j]; j++)
3395 (void)j; /* Quiet lint */
3396 new_holdp = (assignedhd_t **)alloc(SIZEOF(assignedhd_t*)*(j+c+1));
3397 if (sched(diskp)->holdp) {
3398 memcpy(new_holdp, sched(diskp)->holdp, j * SIZEOF(*new_holdp));
3399 amfree(sched(diskp)->holdp);
3401 sched(diskp)->holdp = new_holdp;
3405 if( j > 0 ) { /* This is a request for additional diskspace. See if we can
3406 * merge assignedhd_t's */
3408 if( sched(diskp)->holdp[j-1]->disk == holdp[0]->disk ) { /* Yes! */
3409 sched(diskp)->holdp[j-1]->reserved += holdp[0]->reserved;
3410 holdp[0]->disk->allocated_space += holdp[0]->reserved;
3411 size = (holdp[0]->reserved>size) ? (off_t)0 : size-holdp[0]->reserved;
3412 qname = quote_string(diskp->name);
3413 hold_debug(1, _("assign_holdingdisk: merging holding disk %s to disk %s:%s, add %lld for reserved %lld, left %lld\n"),
3414 holdingdisk_get_diskdir(
3415 sched(diskp)->holdp[j-1]->disk->hdisk),
3416 diskp->host->hostname, qname,
3417 (long long)holdp[0]->reserved,
3418 (long long)sched(diskp)->holdp[j-1]->reserved,
3427 /* copy assignedhd_s to sched(diskp), adjust allocated_space */
3428 for( ; holdp[i]; i++ ) {
3429 holdp[i]->destname = newvstralloc( holdp[i]->destname,
3430 holdingdisk_get_diskdir(holdp[i]->disk->hdisk), "/",
3431 hd_driver_timestamp, "/",
3432 diskp->host->hostname, ".",
3435 sched(diskp)->holdp[j++] = holdp[i];
3436 holdp[i]->disk->allocated_space += holdp[i]->reserved;
3437 size = (holdp[i]->reserved > size) ? (off_t)0 :
3438 (size - holdp[i]->reserved);
3439 qname = quote_string(diskp->name);
3441 _("assign_holdingdisk: %d assigning holding disk %s to disk %s:%s, reserved %lld, left %lld\n"),
3442 i, holdingdisk_get_diskdir(holdp[i]->disk->hdisk),
3443 diskp->host->hostname, qname,
3444 (long long)holdp[i]->reserved,
3447 holdp[i] = NULL; /* so it doesn't get free()d... */
3449 sched(diskp)->holdp[j] = NULL;
3460 assignedhd_t **holdp;
3461 off_t total = (off_t)0;
3464 char *qname, *hqname, *qdest;
3466 (void)cmd; /* Quiet unused parameter warning */
3468 qname = quote_string(diskp->name);
3469 qdest = quote_string(sched(diskp)->destname);
3470 hold_debug(1, _("adjust_diskspace: %s:%s %s\n"),
3471 diskp->host->hostname, qname, qdest);
3473 holdp = sched(diskp)->holdp;
3475 assert(holdp != NULL);
3477 for( i = 0; holdp[i]; i++ ) { /* for each allocated disk */
3478 diff = holdp[i]->used - holdp[i]->reserved;
3479 total += holdp[i]->used;
3480 holdp[i]->disk->allocated_space += diff;
3481 hqname = quote_string(holdingdisk_name(holdp[i]->disk->hdisk));
3482 hold_debug(1, _("adjust_diskspace: hdisk %s done, reserved %lld used %lld diff %lld alloc %lld dumpers %d\n"),
3483 holdingdisk_name(holdp[i]->disk->hdisk),
3484 (long long)holdp[i]->reserved,
3485 (long long)holdp[i]->used,
3487 (long long)holdp[i]->disk->allocated_space,
3488 holdp[i]->disk->allocated_dumpers );
3489 holdp[i]->reserved += diff;
3493 sched(diskp)->act_size = total;
3495 hold_debug(1, _("adjust_diskspace: after: disk %s:%s used %lld\n"),
3496 diskp->host->hostname, qname,
3497 (long long)sched(diskp)->act_size);
3506 assignedhd_t **holdp;
3509 holdp = sched(diskp)->holdp;
3511 assert(holdp != NULL);
3513 for( i = 0; holdp[i]; i++ ) { /* for each disk */
3514 /* find all files of this dump on that disk, and subtract their
3515 * reserved sizes from the disk's allocated space
3517 holdp[i]->disk->allocated_space -= holdp[i]->used;
3520 holding_file_unlink(holdp[0]->destname); /* no need for the entire list,
3521 * because holding_file_unlink
3522 * will walk through all files
3523 * using cont_filename */
3524 free_assignedhd(sched(diskp)->holdp);
3525 sched(diskp)->holdp = NULL;
3526 sched(diskp)->act_size = (off_t)0;
3529 static assignedhd_t **
3536 char buffer[DISK_BLOCK_BYTES];
3538 assignedhd_t **result;
3541 char dirname[1000], *ch;
3543 char *filename = destname;
3545 memset(buffer, 0, sizeof(buffer));
3546 used = alloc(SIZEOF(off_t) * num_holdalloc);
3547 for(i=0;i<num_holdalloc;i++)
3549 result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3551 while(filename != NULL && filename[0] != '\0') {
3552 strncpy(dirname, filename, 999);
3554 ch = strrchr(dirname,'/');
3557 ch = strrchr(dirname,'/');
3564 g_fprintf(stderr,_("build_diskspace: bogus filename '%s'\n"), filename);
3570 for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3571 if(strcmp(dirname, holdingdisk_get_diskdir(ha->hdisk))==0) {
3576 if(stat(filename, &finfo) == -1) {
3577 g_fprintf(stderr, _("stat %s: %s\n"), filename, strerror(errno));
3578 finfo.st_size = (off_t)0;
3580 used[j] += ((off_t)finfo.st_size+(off_t)1023)/(off_t)1024;
3581 if((fd = open(filename,O_RDONLY)) == -1) {
3582 g_fprintf(stderr,_("build_diskspace: open of %s failed: %s\n"),
3583 filename, strerror(errno));
3588 if ((buflen = full_read(fd, buffer, SIZEOF(buffer))) > 0) {;
3589 parse_file_header(buffer, &file, buflen);
3592 filename = file.cont_filename;
3595 for(j = 0, i=0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3596 if(used[j] != (off_t)0) {
3597 result[i] = alloc(SIZEOF(assignedhd_t));
3598 result[i]->disk = ha;
3599 result[i]->reserved = used[j];
3600 result[i]->used = used[j];
3601 result[i]->destname = stralloc(destname);
3619 g_printf(_("driver: hdisk-state time %s"), time_str);
3621 for(ha = holdalloc, dsk = 0; ha != NULL; ha = ha->next, dsk++) {
3622 diff = ha->disksize - ha->allocated_space;
3623 g_printf(_(" hdisk %d: free %lld dumpers %d"), dsk,
3624 (long long)diff, ha->allocated_dumpers);
3633 time_t save_timestamp = sched(dp)->timestamp;
3634 /* setting timestamp to 0 removes the current level from the
3635 * database, so that we ensure that it will not be bumped to the
3636 * next level on the next run. If we didn't do this, dumpdates or
3637 * gnutar-lists might have been updated already, and a bumped
3638 * incremental might be created. */
3639 sched(dp)->timestamp = 0;
3640 update_info_dumper(dp, (off_t)-1, (off_t)-1, (time_t)-1);
3641 sched(dp)->timestamp = save_timestamp;
3644 /* ------------------- */
3653 for(len = 0, p = q.head; p != NULL; len++, p = p->next)
3654 (void)len; /* Quiet lint */
3659 short_dump_state(void)
3664 wall_time = walltime_str(curclock());
3666 g_printf(_("driver: state time %s "), wall_time);
3667 g_printf(_("free kps: %lu space: %lld taper: "),
3669 (long long)free_space());
3670 if(degraded_mode) g_printf(_("DOWN"));
3674 for(taper = tapetable; taper < tapetable+conf_taper_parallel_write;
3676 if (taper->state & TAPER_STATE_DUMP_TO_TAPE ||
3677 taper->state & TAPER_STATE_FILE_TO_TAPE)
3681 g_printf(_("writing"));
3683 g_printf(_("idle"));
3686 for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
3687 g_printf(_(" idle-dumpers: %d"), nidle);
3688 g_printf(_(" qlen tapeq: %d"), queue_length(tapeq));
3689 g_printf(_(" runq: %d"), queue_length(runq));
3690 g_printf(_(" roomq: %d"), queue_length(roomq));
3691 g_printf(_(" wakeup: %d"), (int)sleep_time);
3692 g_printf(_(" driver-idle: %s\n"), _(idle_strings[idle_reason]));
3693 interface_state(wall_time);
3694 holdingdisk_state(wall_time);
3701 char **why_no_new_tape)
3703 TapeAction result = TAPE_ACTION_NO_ACTION;
3712 off_t dump_to_disk_size;
3713 int dump_to_disk_terminated;
3714 int nb_taper_active = nb_sent_new_tape;
3715 int nb_taper_flushing = 0;
3716 off_t data_next_tape = 0;
3717 off_t data_free = 0;
3718 off_t data_lost = 0;
3719 off_t data_lost_next_tape = 0;
3722 for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
3723 if (dumper->busy && !sched(dumper->dp)->taper)
3724 dumpers_size += sched(dumper->dp)->est_size;
3726 driver_debug(1, _("dumpers_size: %lld\n"), (long long)dumpers_size);
3729 for(dp = runq.head; dp != NULL; dp = dp->next) {
3730 runq_size += sched(dp)->est_size;
3732 driver_debug(1, _("runq_size: %lld\n"), (long long)runq_size);
3735 for(dp = directq.head; dp != NULL; dp = dp->next) {
3736 directq_size += sched(dp)->est_size;
3738 driver_debug(1, _("directq_size: %lld\n"), (long long)directq_size);
3740 tapeq_size = directq_size;
3741 for(dp = tapeq.head; dp != NULL; dp = dp->next) {
3742 tapeq_size += sched(dp)->act_size;
3745 for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3747 if (taper1->state & TAPER_STATE_FILE_TO_TAPE ||
3748 taper1->state & TAPER_STATE_DUMP_TO_TAPE) {
3749 nb_taper_flushing++;
3753 /* Add what is currently written to tape and in the go. */
3754 for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3756 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3757 tapeq_size -= taper1->left;
3761 if (taper1->dumper) {
3762 data_to_go = sched(taper1->disk)->est_size - taper1->written;
3764 data_to_go = sched(taper1->disk)->act_size - taper1->written;
3766 if (data_to_go > taper1->left) {
3767 data_next_tape += data_to_go - taper1->left;
3768 data_lost += taper1->written + taper1->left;
3770 data_free += taper1->left - data_to_go;
3772 tapeq_size += data_to_go;
3775 data_lost_next_tape = tape_length + data_free - data_next_tape - runq_size - directq_size - tapeq_size;
3776 driver_debug(1, _("data_lost: %lld\n"), (long long)data_lost);
3777 driver_debug(1, _("data_free: %lld\n"), (long long)data_free);
3778 driver_debug(1, _("data_next_tape: %lld\n"), (long long)data_next_tape);
3779 driver_debug(1, _("data_lost_next_tape: %lld\n"), (long long)data_lost_next_tape);
3781 driver_debug(1, _("tapeq_size: %lld\n"), (long long)tapeq_size);
3783 sched_size = runq_size + directq_size + tapeq_size + dumpers_size;
3784 driver_debug(1, _("sched_size: %lld\n"), (long long)sched_size);
3786 dump_to_disk_size = dumpers_size + runq_size + directq_size;
3787 driver_debug(1, _("dump_to_disk_size: %lld\n"), (long long)dump_to_disk_size);
3789 dump_to_disk_terminated = schedule_done && dump_to_disk_size == 0;
3791 for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
3793 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3798 // Changing conditionals can produce a driver hang, take care.
3800 // when to start writting to a new tape
3801 if (taper->state & TAPER_STATE_TAPE_REQUESTED) {
3802 if (current_tape >= conf_runtapes && taper_nb_scan_volume == 0 &&
3803 nb_taper_active == 0) {
3804 *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3805 "does not allow additional tapes"), current_tape, conf_runtapes);
3806 result |= TAPE_ACTION_NO_NEW_TAPE;
3807 } else if (current_tape < conf_runtapes &&
3808 taper_nb_scan_volume == 0 &&
3809 ((flush_threshold_dumped < tapeq_size &&
3810 flush_threshold_scheduled < sched_size) ||
3811 (data_lost > data_lost_next_tape) ||
3812 nb_taper_active == 0) &&
3813 (last_started_taper == NULL ||
3814 last_started_taper == taper)) {
3815 result |= TAPE_ACTION_SCAN;
3817 result |= TAPE_ACTION_MOVE;
3819 } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3820 ((taper->state & TAPER_STATE_DUMP_TO_TAPE) || // for dump to tape
3821 !empty(directq) || // if a dle is waiting for a dump to tape
3822 !empty(roomq) || // holding disk constraint
3823 idle_reason == IDLE_NO_DISKSPACE || // holding disk constraint
3824 (flush_threshold_dumped < tapeq_size && // flush-threshold-dumped &&
3825 flush_threshold_scheduled < sched_size) || // flush-threshold-scheduled
3826 (data_lost > data_lost_next_tape) ||
3827 (taperflush < tapeq_size && // taperflush
3828 (force_flush == 1 || // if force_flush
3829 dump_to_disk_terminated)) // or all dump to disk terminated
3831 result |= TAPE_ACTION_NEW_TAPE;
3832 // when to stop using new tape
3833 } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3834 (taperflush >= tapeq_size && // taperflush criteria
3835 (force_flush == 1 || // if force_flush
3836 dump_to_disk_terminated)) // or all dump to disk
3838 if (nb_taper_active <= 0) {
3839 if (current_tape >= conf_runtapes) {
3840 *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3841 "does not allow additional tapes"), current_tape, conf_runtapes);
3843 *why_no_new_tape = _("taperflush criteria not met");
3845 result |= TAPE_ACTION_NO_NEW_TAPE;
3849 // when to start a flush
3850 if (taper->state & TAPER_STATE_IDLE) {
3851 if (!degraded_mode && (!empty(tapeq) || !empty(directq)) &&
3852 (taper->state & TAPER_STATE_TAPE_STARTED || // tape already started
3853 !empty(roomq) || // holding disk constraint
3854 idle_reason == IDLE_NO_DISKSPACE || // holding disk constraint
3855 (flush_threshold_dumped < tapeq_size && // flush-threshold-dumped &&
3856 flush_threshold_scheduled < sched_size) || // flush-threshold-scheduled
3857 (force_flush == 1 && taperflush < tapeq_size))) { // taperflush if force_flush
3859 if (nb_taper_flushing == 0) {
3860 result |= TAPE_ACTION_START_A_FLUSH;
3862 result |= TAPE_ACTION_START_A_FLUSH_FIT;
3870 no_taper_flushing(void)
3874 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
3876 if (taper->state & TAPER_STATE_FILE_TO_TAPE)
3891 g_printf("================\n");
3892 g_printf(_("driver state at time %s: %s\n"), walltime_str(curclock()), str);
3893 g_printf(_("free kps: %lu, space: %lld\n"),
3895 (long long)free_space());
3896 if(degraded_mode) g_printf(_("taper: DOWN\n"));
3897 else if(taper->status == TAPER_IDLE) g_printf(_("taper: idle\n"));
3898 else g_printf(_("taper: writing %s:%s.%d est size %lld\n"),
3899 taper->disk->host->hostname, taper->disk->name,
3900 sched(taper->disk)->level,
3901 (long long)sched(taper->disk)->est_size);
3902 for(i = 0; i < inparallel; i++) {
3903 dp = dmptable[i].dp;
3904 if(!dmptable[i].busy)
3905 g_printf(_("%s: idle\n"), dmptable[i].name);
3907 qname = quote_string(dp->name);
3908 g_printf(_("%s: dumping %s:%s.%d est kps %d size %lld time %lu\n"),
3909 dmptable[i].name, dp->host->hostname, qname, sched(dp)->level,
3910 sched(dp)->est_kps, (long long)sched(dp)->est_size, sched(dp)->est_time);
3913 dump_queue("TAPE", tapeq, 5, stdout);
3914 dump_queue("ROOM", roomq, 5, stdout);
3915 dump_queue("RUN ", runq, 5, stdout);
3916 g_printf("================\n");