2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 1991-1998 University of Maryland at College Park
6 * Permission to use, copy, modify, distribute, and sell this software and its
7 * documentation for any purpose is hereby granted without fee, provided that
8 * the above copyright notice appear in all copies and that both that
9 * copyright notice and this permission notice appear in supporting
10 * documentation, and that the name of U.M. not be used in advertising or
11 * publicity pertaining to distribution of the software without specific,
12 * written prior permission. U.M. makes no representations about the
13 * suitability of this software for any purpose. It is provided "as is"
14 * without express or implied warranty.
16 * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
18 * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
19 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
20 * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
21 * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
23 * Authors: the Amanda Development Team. Its members are listed in a
24 * file named AUTHORS, in the root directory of this distribution.
27 * $Id: driver.c 6512 2007-05-24 17:00:24Z ian $
29 * controlling process for the Amanda backup system
33 * XXX possibly modify tape queue to be cognizant of how much room is left on
34 * tape. Probably not effective though, should do this in planner.
48 #include "server_util.h"
49 #include "timestamp.h"
51 #define driver_debug(i, ...) do { \
52 if ((i) <= debug_driver) { \
53 dbprintf(__VA_ARGS__); \
57 #define hold_debug(i, ...) do { \
58 if ((i) <= debug_holding) { \
59 dbprintf(__VA_ARGS__); \
63 static disklist_t waitq; // dle waiting estimate result
64 static disklist_t runq; // dle waiting to be dumped to holding disk
65 static disklist_t directq; // dle waiting to be dumped directly to tape
66 static disklist_t tapeq; // dle on holding disk waiting to be written
68 static disklist_t roomq; // dle waiting for more space on holding disk
69 static int pending_aborts;
70 static int degraded_mode;
71 static off_t reserved_space;
72 static off_t total_disksize;
73 static char *dumper_program;
74 static char *chunker_program;
75 static int inparallel;
76 static int nodump = 0;
77 static off_t tape_length = (off_t)0;
78 static int current_tape = 0;
79 static int conf_taperalgo;
80 static int conf_taper_parallel_write;
81 static int conf_runtapes;
82 static time_t sleep_time;
83 static int idle_reason;
84 static char *driver_timestamp;
85 static char *hd_driver_timestamp;
86 static am_host_t *flushhost = NULL;
87 static int need_degraded=0;
88 static holdalloc_t *holdalloc;
89 static int num_holdalloc;
90 static event_handle_t *dumpers_ev_time = NULL;
91 static event_handle_t *flush_ev_read = NULL;
92 static event_handle_t *schedule_ev_read = NULL;
93 static int conf_flush_threshold_dumped;
94 static int conf_flush_threshold_scheduled;
95 static int conf_taperflush;
96 static off_t flush_threshold_dumped;
97 static off_t flush_threshold_scheduled;
98 static off_t taperflush;
99 static int schedule_done; // 1 if we don't wait for a
100 // schedule from the planner
101 static int force_flush; // All dump are terminated, we
102 // must now respect taper_flush
103 static int taper_nb_scan_volume = 0;
104 static int nb_sent_new_tape = 0;
105 static int taper_started = 0;
106 static taper_t *last_started_taper;
108 static int wait_children(int count);
109 static void wait_for_children(void);
110 static void allocate_bandwidth(netif_t *ip, unsigned long kps);
111 static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp);
112 static void adjust_diskspace(disk_t *diskp, cmd_t cmd);
113 static void delete_diskspace(disk_t *diskp);
114 static assignedhd_t **build_diskspace(char *destname);
115 static int client_constrained(disk_t *dp);
116 static void deallocate_bandwidth(netif_t *ip, unsigned long kps);
117 static void dump_schedule(disklist_t *qp, char *str);
118 static assignedhd_t **find_diskspace(off_t size, int *cur_idle,
119 assignedhd_t *preferred);
120 static unsigned long free_kps(netif_t *ip);
121 static off_t free_space(void);
122 static void dumper_chunker_result(disk_t *dp);
123 static void dumper_taper_result(disk_t *dp);
124 static void file_taper_result(disk_t *dp);
125 static void handle_dumper_result(void *);
126 static void handle_chunker_result(void *);
127 static void handle_dumpers_time(void *);
128 static void handle_taper_result(void *);
130 static void holdingdisk_state(char *time_str);
131 static taper_t *idle_taper(void);
132 static taper_t *taper_from_name(char *name);
133 static void interface_state(char *time_str);
134 static int queue_length(disklist_t q);
135 static void read_flush(void *cookie);
136 static void read_schedule(void *cookie);
137 static void short_dump_state(void);
138 static void startaflush(void);
139 static void start_degraded_mode(disklist_t *queuep);
140 static void start_some_dumps(disklist_t *rq);
141 static void continue_port_dumps(void);
142 static void update_failed_dump(disk_t *);
143 static int all_taper_idle(void);
146 TAPE_ACTION_NO_ACTION = 0,
147 TAPE_ACTION_SCAN = (1 << 0),
148 TAPE_ACTION_NEW_TAPE = (1 << 1),
149 TAPE_ACTION_NO_NEW_TAPE = (1 << 2),
150 TAPE_ACTION_START_A_FLUSH = (1 << 3),
151 TAPE_ACTION_START_A_FLUSH_FIT = (1 << 4),
152 TAPE_ACTION_MOVE = (1 << 5)
155 static TapeAction tape_action(taper_t *taper, char **why_no_new_tape);
157 static const char *idle_strings[] = {
160 #define IDLE_NO_DUMPERS 1
162 #define IDLE_START_WAIT 2
164 #define IDLE_NO_HOLD 3
166 #define IDLE_CLIENT_CONSTRAINED 4
167 T_("client-constrained"),
168 #define IDLE_NO_BANDWIDTH 5
170 #define IDLE_NO_DISKSPACE 6
184 struct fs_usage fsusage;
187 unsigned long reserve = 100;
189 char **result_argv = NULL;
196 config_overrides_t *cfg_ovr = NULL;
197 char *cfg_opt = NULL;
198 holdalloc_t *ha, *ha_last;
199 find_result_t *holding_files;
200 disklist_t holding_disklist = { NULL, NULL };
201 int no_taper = FALSE;
204 * Configure program for internationalization:
205 * 1) Only set the message locale for now.
206 * 2) Set textdomain for all amanda related programs to "amanda"
207 * We don't want to be forced to support dozens of message catalogs.
209 setlocale(LC_MESSAGES, "C");
210 textdomain("amanda");
214 setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0);
215 setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0);
219 dbopen(DBG_SUBDIR_SERVER);
221 atexit(wait_for_children);
223 /* Don't die when child closes pipe */
224 signal(SIGPIPE, SIG_IGN);
226 add_amanda_log_handler(amanda_log_stderr);
227 add_amanda_log_handler(amanda_log_trace_log);
231 cfg_ovr = extract_commandline_config_overrides(&argc, &argv);
235 set_config_overrides(cfg_ovr);
236 config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
238 conf_diskfile = config_dir_relative(getconf_str(CNF_DISKFILE));
239 read_diskfile(conf_diskfile, &origq);
240 disable_skip_disk(&origq);
241 amfree(conf_diskfile);
243 if (config_errors(NULL) >= CFGERR_WARNINGS) {
244 config_print_errors();
245 if (config_errors(NULL) >= CFGERR_ERRORS) {
246 g_critical(_("errors processing config file"));
250 log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
251 g_printf(_("%s: pid %ld executable %s version %s\n"),
252 get_pname(), (long) getpid(), argv[0], VERSION);
255 if(strcmp(argv[2], "nodump") == 0) {
263 if (strcmp(argv[2], "--no-taper") == 0) {
270 safe_cd(); /* do this *after* config_init */
272 check_running_as(RUNNING_AS_DUMPUSER);
274 dbrename(get_config_name(), DBG_SUBDIR_SERVER);
276 /* load DLEs from the holding disk, in case there's anything to flush there */
277 search_holding_disk(&holding_files, &holding_disklist);
278 /* note that the dumps are added to the global disklist, so we need not consult
279 * holding_files or holding_disklist after this */
281 amfree(driver_timestamp);
282 /* read timestamp from stdin */
283 while ((line = agets(stdin)) != NULL) {
288 if ( line == NULL ) {
289 error(_("Did not get DATE line from planner"));
292 driver_timestamp = alloc(15);
293 strncpy(driver_timestamp, &line[5], 14);
294 driver_timestamp[14] = '\0';
296 log_add(L_START,_("date %s"), driver_timestamp);
298 gethostname(hostname, SIZEOF(hostname));
299 log_add(L_STATS,_("hostname %s"), hostname);
301 /* check that we don't do many dump in a day and usetimestamps is off */
302 if(strlen(driver_timestamp) == 8) {
304 char *conf_logdir = getconf_str(CNF_LOGDIR);
305 char *logfile = vstralloc(conf_logdir, "/log.",
306 driver_timestamp, ".0", NULL);
307 char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.",
308 driver_timestamp, ".0", NULL);
309 if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) {
310 log_add(L_WARNING, _("WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day."));
315 hd_driver_timestamp = get_timestamp_from_time(0);
318 hd_driver_timestamp = stralloc(driver_timestamp);
321 taper_program = vstralloc(amlibexecdir, "/", "taper", NULL);
322 dumper_program = vstralloc(amlibexecdir, "/", "dumper", NULL);
323 chunker_program = vstralloc(amlibexecdir, "/", "chunker", NULL);
325 conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO);
326 conf_taper_parallel_write = getconf_int(CNF_TAPER_PARALLEL_WRITE);
327 conf_tapetype = getconf_str(CNF_TAPETYPE);
328 conf_runtapes = getconf_int(CNF_RUNTAPES);
329 if (conf_taper_parallel_write > conf_runtapes) {
330 conf_taper_parallel_write = conf_runtapes;
332 tape = lookup_tapetype(conf_tapetype);
333 tape_length = tapetype_get_length(tape);
334 g_printf("driver: tape size %lld\n", (long long)tape_length);
335 conf_flush_threshold_dumped = getconf_int(CNF_FLUSH_THRESHOLD_DUMPED);
336 conf_flush_threshold_scheduled = getconf_int(CNF_FLUSH_THRESHOLD_SCHEDULED);
337 conf_taperflush = getconf_int(CNF_TAPERFLUSH);
339 flush_threshold_dumped = (conf_flush_threshold_dumped * tape_length) / 100;
340 flush_threshold_scheduled = (conf_flush_threshold_scheduled * tape_length) / 100;
341 taperflush = (conf_taperflush *tape_length) / 100;
343 driver_debug(1, _("flush-threshold-dumped: %lld\n"), (long long)flush_threshold_dumped);
344 driver_debug(1, _("flush-threshold-scheduled: %lld\n"), (long long)flush_threshold_scheduled);
345 driver_debug(1, _("taperflush: %lld\n"), (long long)taperflush);
347 /* set up any configuration-dependent variables */
349 inparallel = getconf_int(CNF_INPARALLEL);
351 reserve = (unsigned long)getconf_int(CNF_RESERVE);
353 total_disksize = (off_t)0;
356 for (il = getconf_identlist(CNF_HOLDINGDISK), dsk = 0;
358 il = il->next, dsk++) {
359 hdp = lookup_holdingdisk(il->data);
360 ha = alloc(SIZEOF(holdalloc_t));
363 /* link the list in the same order as getconf_holdingdisks's results */
372 ha->allocated_dumpers = 0;
373 ha->allocated_space = (off_t)0;
374 ha->disksize = holdingdisk_get_disksize(hdp);
377 if(get_fs_usage(holdingdisk_get_diskdir(hdp), NULL, &fsusage) == -1
378 || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) {
379 log_add(L_WARNING, _("WARNING: ignoring holding disk %s: %s\n"),
380 holdingdisk_get_diskdir(hdp), strerror(errno));
385 /* do the division first to avoid potential integer overflow */
386 if (fsusage.fsu_bavail_top_bit_set)
389 kb_avail = fsusage.fsu_bavail / 1024 * fsusage.fsu_blocksize;
391 if(ha->disksize > (off_t)0) {
392 if(ha->disksize > kb_avail) {
394 _("WARNING: %s: %lld KB requested, "
395 "but only %lld KB available."),
396 holdingdisk_get_diskdir(hdp),
397 (long long)ha->disksize,
398 (long long)kb_avail);
399 ha->disksize = kb_avail;
402 /* ha->disksize is negative; use all but that amount */
403 else if(kb_avail < -ha->disksize) {
405 _("WARNING: %s: not %lld KB free."),
406 holdingdisk_get_diskdir(hdp),
407 (long long)-ha->disksize);
408 ha->disksize = (off_t)0;
412 ha->disksize += kb_avail;
414 g_printf(_("driver: adding holding disk %d dir %s size %lld chunksize %lld\n"),
415 dsk, holdingdisk_get_diskdir(hdp),
416 (long long)ha->disksize,
417 (long long)(holdingdisk_get_chunksize(hdp)));
419 newdir = newvstralloc(newdir,
420 holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp,
422 if(!mkholdingdir(newdir)) {
423 ha->disksize = (off_t)0;
425 total_disksize += ha->disksize;
428 reserved_space = total_disksize * (off_t)(reserve / 100);
430 g_printf(_("reserving %lld out of %lld for degraded-mode dumps\n"),
431 (long long)reserved_space, (long long)free_space());
435 if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS;
437 /* taper takes a while to get going, so start it up right away */
440 startup_tape_process(taper_program, conf_taper_parallel_write, no_taper);
442 /* fire up the dumpers now while we are waiting */
443 if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp);
446 * Read schedule from stdin. Usually, this is a pipe from planner,
447 * so the effect is that we wait here for the planner to
448 * finish, but meanwhile the taper is rewinding the tape, reading
449 * the label, checking it, writing a new label and all that jazz
450 * in parallel with the planner.
462 taper_nb_wait_reply = 0;
465 if (no_taper || conf_runtapes <= 0) {
466 taper_started = 1; /* we'll pretend the taper started and failed immediately */
469 tapetable[0].state = TAPER_STATE_INIT;
470 taper_nb_wait_reply++;
471 taper_nb_scan_volume++;
472 taper_ev_read = event_register(taper_fd, EV_READFD,
473 handle_taper_result, NULL);
474 taper_cmd(START_TAPER, NULL, tapetable[0].name, 0, driver_timestamp);
477 flush_ev_read = event_register((event_id_t)0, EV_READFD, read_flush, NULL);
479 log_add(L_STATS, _("startup time %s"), walltime_str(curclock()));
481 g_printf(_("driver: start time %s inparallel %d bandwidth %lu diskspace %lld "), walltime_str(curclock()), inparallel,
482 free_kps(NULL), (long long)free_space());
483 g_printf(_(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n"),
484 "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo),
485 getconf_str(CNF_DUMPORDER));
488 schedule_done = nodump;
496 /* mv runq to directq */
497 while (!empty(runq)) {
498 diskp = dequeue_disk(&runq);
499 headqueue_disk(&directq, diskp);
502 /* log error for any remaining dumps */
503 while(!empty(directq)) {
504 diskp = dequeue_disk(&directq);
506 if (diskp->to_holdingdisk == HOLD_REQUIRED) {
507 char *qname = quote_string(diskp->name);
508 log_add(L_FAIL, "%s %s %s %d [%s]",
509 diskp->host->hostname, qname, sched(diskp)->datestamp,
511 _("can't dump required holdingdisk"));
514 else if (!degraded_mode) {
515 char *qname = quote_string(diskp->name);
516 log_add(L_FAIL, "%s %s %s %d [%s]",
517 diskp->host->hostname, qname, sched(diskp)->datestamp,
519 _("can't dump in non degraded mode"));
523 char *qname = quote_string(diskp->name);
524 log_add(L_FAIL, "%s %s %s %d [%s]",
525 diskp->host->hostname, qname, sched(diskp)->datestamp,
528 _("can't do degraded dump without holding disk") :
529 diskp->to_holdingdisk != HOLD_NEVER ?
530 _("out of holding space in degraded mode") :
531 _("can't dump 'holdingdisk never' dle in degraded mode"));
536 short_dump_state(); /* for amstatus */
538 g_printf(_("driver: QUITTING time %s telling children to quit\n"),
539 walltime_str(curclock()));
543 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
545 dumper_cmd(dumper, QUIT, NULL, NULL);
550 taper_cmd(QUIT, NULL, NULL, 0, NULL);
553 /* wait for all to die */
557 holding_cleanup(NULL, NULL);
561 check_unfree_serial();
562 g_printf(_("driver: FINISHED time %s\n"), walltime_str(curclock()));
564 log_add(L_FINISH,_("date %s time %s"), driver_timestamp, walltime_str(curclock()));
565 log_add(L_INFO, "pid-done %ld", (long)getpid());
566 amfree(driver_timestamp);
568 amfree(dumper_program);
569 amfree(taper_program);
571 g_strfreev(result_argv);
578 /* sleep up to count seconds, and wait for terminating child process */
579 /* if sleep is negative, this function will not timeout */
580 /* exit once all child process are finished or the timout expired */
581 /* return 0 if no more children to wait */
582 /* return 1 if some children are still alive */
584 wait_children(int count)
596 pid = waitpid((pid_t)-1, &retstat, WNOHANG);
600 if (! WIFEXITED(retstat)) {
602 code = WTERMSIG(retstat);
603 } else if (WEXITSTATUS(retstat) != 0) {
605 code = WEXITSTATUS(retstat);
608 for (dumper = dmptable; dumper < dmptable + inparallel;
610 if (pid == dumper->pid) {
611 who = stralloc(dumper->name);
615 if (dumper->chunker && pid == dumper->chunker->pid) {
616 who = stralloc(dumper->chunker->name);
617 dumper->chunker->pid = -1;
621 if (who == NULL && pid == taper_pid) {
622 who = stralloc("taper");
625 if(what != NULL && who == NULL) {
626 who = stralloc("unknown");
629 log_add(L_WARNING, _("%s pid %u exited with %s %d\n"), who,
630 (unsigned)pid, what, code);
631 g_printf(_("driver: %s pid %u exited with %s %d\n"), who,
632 (unsigned)pid, what, code);
636 } while (pid > 0 || wait_errno == EINTR);
641 } while ((errno != ECHILD) && (count != 0));
642 return (errno != ECHILD);
646 kill_children(int signal)
651 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
652 if (!dumper->down && dumper->pid > 1) {
653 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
654 dumper->name, (unsigned)dumper->pid);
655 if (kill(dumper->pid, signal) == -1 && errno == ESRCH) {
657 dumper->chunker->pid = 0;
659 if (dumper->chunker && dumper->chunker->pid > 1) {
660 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
661 dumper->chunker->name,
662 (unsigned)dumper->chunker->pid);
663 if (kill(dumper->chunker->pid, signal) == -1 &&
665 dumper->chunker->pid = 0;
672 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
673 "taper", (unsigned)taper_pid);
674 if (kill(taper_pid, signal) == -1 && errno == ESRCH)
680 wait_for_children(void)
685 for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
686 if (dumper->pid > 1 && dumper->fd >= 0) {
687 dumper_cmd(dumper, QUIT, NULL, NULL);
688 if (dumper->chunker && dumper->chunker->pid > 1 &&
689 dumper->chunker->fd >= 0)
690 chunker_cmd(dumper->chunker, QUIT, NULL, NULL);
695 if(taper_pid > 1 && taper_fd > 0) {
696 taper_cmd(QUIT, NULL, NULL, 0, NULL);
699 if(wait_children(60) == 0)
702 kill_children(SIGHUP);
703 if(wait_children(60) == 0)
706 kill_children(SIGKILL);
707 if(wait_children(-1) == 0)
712 static void startaflush_tape(taper_t *taper);
719 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
721 if (!(taper->state & TAPER_STATE_DONE) &&
722 taper->state & TAPER_STATE_WAIT_FOR_TAPE) {
723 startaflush_tape(taper);
726 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
728 if (!(taper->state & TAPER_STATE_DONE) &&
729 taper->state & TAPER_STATE_TAPE_REQUESTED) {
730 startaflush_tape(taper);
733 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
735 if (!(taper->state & TAPER_STATE_DONE) &&
736 taper->state & TAPER_STATE_INIT) {
737 startaflush_tape(taper);
740 for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
742 if (!(taper->state & TAPER_STATE_DONE) &&
743 taper->state & TAPER_STATE_IDLE) {
744 startaflush_tape(taper);
756 off_t extra_tapes_size = 0;
759 TapeAction result_tape_action;
760 char *why_no_new_tape = NULL;
763 result_tape_action = tape_action(taper, &why_no_new_tape);
765 if (result_tape_action & TAPE_ACTION_SCAN) {
766 taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
767 taper->state |= TAPER_STATE_WAIT_FOR_TAPE;
768 taper_nb_scan_volume++;
769 taper_cmd(START_SCAN, taper->disk, NULL, 0, NULL);
770 } else if (result_tape_action & TAPE_ACTION_NEW_TAPE) {
771 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
772 taper->state |= TAPER_STATE_WAIT_NEW_TAPE;
774 taper_cmd(NEW_TAPE, taper->disk, NULL, 0, NULL);
775 } else if (result_tape_action & TAPE_ACTION_NO_NEW_TAPE) {
776 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
777 taper_cmd(NO_NEW_TAPE, taper->disk, why_no_new_tape, 0, NULL);
778 taper->state |= TAPER_STATE_DONE;
779 start_degraded_mode(&runq);
780 } else if (result_tape_action & TAPE_ACTION_MOVE) {
781 taper_t *taper1 = idle_taper();
783 taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
784 taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
785 taper_cmd(TAKE_SCRIBE_FROM, taper->disk, taper1->name, 0 , NULL);
786 taper1->state = TAPER_STATE_DEFAULT;
787 taper->state |= TAPER_STATE_TAPE_STARTED;
788 taper->left = taper1->left;
789 if (last_started_taper == taper1) {
790 last_started_taper = taper;
795 if (!degraded_mode &&
796 taper->state & TAPER_STATE_IDLE &&
798 (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
799 result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT)) {
801 int taperalgo = conf_taperalgo;
802 if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
803 if (taperalgo == ALGO_FIRST)
804 taperalgo = ALGO_FIRSTFIT;
805 else if (taperalgo == ALGO_LARGEST)
806 taperalgo = ALGO_LARGESTFIT;
807 else if (taperalgo == ALGO_SMALLEST)
808 taperalgo = ALGO_SMALLESTFIT;
809 else if (taperalgo == ALGO_LAST)
810 taperalgo = ALGO_LASTFIT;
813 extra_tapes_size = tape_length * (off_t)(conf_runtapes - current_tape);
814 for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
816 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
817 extra_tapes_size += taper1->left;
821 extra_tapes_size -= (sched(dp)->act_size - taper1->written);
825 if (taper->state & TAPER_STATE_TAPE_STARTED) {
826 taper_left = taper->left;
828 taper_left = tape_length;
831 datestamp = sched(tapeq.head)->datestamp;
834 dp = dequeue_disk(&tapeq);
838 while (fit != NULL) {
839 if (sched(fit)->act_size <=
840 (fit->splitsize ? extra_tapes_size : taper_left) &&
841 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
849 if(dp) remove_disk(&tapeq, dp);
852 fit = dp = tapeq.head;
853 while (fit != NULL) {
854 if(sched(fit)->act_size > sched(dp)->act_size &&
855 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
860 if(dp) remove_disk(&tapeq, dp);
862 case ALGO_LARGESTFIT:
864 while (fit != NULL) {
865 if(sched(fit)->act_size <=
866 (fit->splitsize ? extra_tapes_size : taper_left) &&
867 (!dp || sched(fit)->act_size > sched(dp)->act_size) &&
868 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
873 if(dp) remove_disk(&tapeq, dp);
876 fit = dp = tapeq.head;
877 while (fit != NULL) {
878 if (sched(fit)->act_size < sched(dp)->act_size &&
879 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
884 if(dp) remove_disk(&tapeq, dp);
886 case ALGO_SMALLESTFIT:
887 fit = dp = tapeq.head;
888 while (fit != NULL) {
889 if (sched(fit)->act_size <=
890 (fit->splitsize ? extra_tapes_size : taper_left) &&
891 (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
892 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
897 if(dp) remove_disk(&tapeq, dp);
901 remove_disk(&tapeq, dp);
905 while (fit != NULL) {
906 if (sched(fit)->act_size <=
907 (fit->splitsize ? extra_tapes_size : taper_left) &&
908 (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
909 strcmp(sched(fit)->datestamp, datestamp) <= 0) {
914 if(dp) remove_disk(&tapeq, dp);
919 taper->dumper = NULL;
920 amfree(taper->input_error);
921 amfree(taper->tape_error);
922 taper->result = LAST_TOK;
923 taper->sendresult = 0;
924 amfree(taper->first_label);
926 taper->state &= ~TAPER_STATE_IDLE;
927 taper->state |= TAPER_STATE_FILE_TO_TAPE;
928 taper->dumper = NULL;
929 qname = quote_string(dp->name);
930 if (taper_nb_wait_reply == 0) {
931 taper_ev_read = event_register(taper_fd, EV_READFD,
932 handle_taper_result, NULL);
934 taper_nb_wait_reply++;
935 sched(dp)->taper = taper;
936 taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level,
937 sched(dp)->datestamp);
938 g_fprintf(stderr,_("driver: startaflush: %s %s %s %lld %lld\n"),
939 taperalgo2str(taperalgo), dp->host->hostname, qname,
940 (long long)sched(taper->disk)->act_size,
941 (long long)taper->left);
954 /* first, check if host is too busy */
956 if(dp->host->inprogress >= dp->host->maxdumps) {
960 /* next, check conflict with other dumps on same spindle */
962 if(dp->spindle == -1) { /* but spindle -1 never conflicts by def. */
966 for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext)
967 if(dp2->inprogress && dp2->spindle == dp->spindle) {
981 int dumper_to_holding,
983 disk_t **delayed_diskp,
984 disk_t **diskp_accept,
985 assignedhd_t ***holdp_accept,
986 off_t extra_tapes_size)
988 assignedhd_t **holdp=NULL;
990 if (diskp->host->start_t > now) {
991 *cur_idle = max(*cur_idle, IDLE_START_WAIT);
992 if (*delayed_diskp == NULL || sleep_time > diskp->host->start_t) {
993 *delayed_diskp = diskp;
994 sleep_time = diskp->host->start_t;
996 } else if(diskp->start_t > now) {
997 *cur_idle = max(*cur_idle, IDLE_START_WAIT);
998 if (*delayed_diskp == NULL || sleep_time > diskp->start_t) {
999 *delayed_diskp = diskp;
1000 sleep_time = diskp->start_t;
1002 } else if (diskp->host->netif->curusage > 0 &&
1003 sched(diskp)->est_kps > free_kps(diskp->host->netif)) {
1004 *cur_idle = max(*cur_idle, IDLE_NO_BANDWIDTH);
1005 } else if (!taper && sched(diskp)->no_space) {
1006 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1007 } else if (!taper && diskp->to_holdingdisk == HOLD_NEVER) {
1008 *cur_idle = max(*cur_idle, IDLE_NO_HOLD);
1009 } else if (extra_tapes_size && sched(diskp)->est_size > extra_tapes_size) {
1010 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1012 } else if (!taper && (holdp =
1013 find_diskspace(sched(diskp)->est_size, cur_idle, NULL)) == NULL) {
1014 *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1015 if (empty(tapeq) && dumper_to_holding == 0 && rq != &directq && all_taper_idle()) {
1016 remove_disk(rq, diskp);
1017 if (diskp->to_holdingdisk != HOLD_REQUIRED) {
1018 enqueue_disk(&directq, diskp);
1019 diskp->to_holdingdisk = HOLD_NEVER;
1021 if (empty(*rq)) force_flush = 1;
1023 } else if (client_constrained(diskp)) {
1024 free_assignedhd(holdp);
1025 *cur_idle = max(*cur_idle, IDLE_CLIENT_CONSTRAINED);
1028 /* disk fits, dump it */
1029 int accept = !*diskp_accept;
1032 case 's': accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1034 case 'S': accept = (sched(diskp)->est_size > sched(*diskp_accept)->est_size);
1036 case 't': accept = (sched(diskp)->est_time < sched(*diskp_accept)->est_time);
1038 case 'T': accept = (sched(diskp)->est_time > sched(*diskp_accept)->est_time);
1040 case 'b': accept = (sched(diskp)->est_kps < sched(*diskp_accept)->est_kps);
1042 case 'B': accept = (sched(diskp)->est_kps > sched(*diskp_accept)->est_kps);
1044 default: log_add(L_WARNING, _("Unknown dumporder character \'%c\', using 's'.\n"),
1046 accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1051 if( !*diskp_accept || !degraded_mode || diskp->priority >= (*diskp_accept)->priority) {
1052 if(*holdp_accept) free_assignedhd(*holdp_accept);
1053 *diskp_accept = diskp;
1054 *holdp_accept = holdp;
1057 free_assignedhd(holdp);
1061 free_assignedhd(holdp);
1070 const time_t now = time(NULL);
1072 disk_t *diskp, *delayed_diskp, *diskp_accept;
1074 assignedhd_t **holdp=NULL, **holdp_accept;
1083 int dumper_to_holding = 0;
1085 /* don't start any actual dumps until the taper is started */
1086 if (!taper_started) return;
1088 idle_reason = IDLE_NO_DUMPERS;
1091 if(dumpers_ev_time != NULL) {
1092 event_release(dumpers_ev_time);
1093 dumpers_ev_time = NULL;
1096 for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1097 if (dumper->busy && dumper->dp->to_holdingdisk != HOLD_NEVER) {
1098 dumper_to_holding++;
1101 for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) {
1103 if( dumper->busy || dumper->down) {
1107 if (dumper->ev_read != NULL) {
1108 event_release(dumper->ev_read);
1109 dumper->ev_read = NULL;
1113 * A potential problem with starting from the bottom of the dump time
1114 * distribution is that a slave host will have both one of the shortest
1115 * and one of the longest disks, so starting its shortest disk first will
1116 * tie up the host and eliminate its longest disk from consideration the
1117 * first pass through. This could cause a big delay in starting that long
1118 * disk, which could drag out the whole night's dumps.
1120 * While starting from the top of the dump time distribution solves the
1121 * above problem, this turns out to be a bad idea, because the big dumps
1122 * will almost certainly pack the holding disk completely, leaving no
1123 * room for even one small dump to start. This ends up shutting out the
1124 * small-end dumpers completely (they stay idle).
1126 * The introduction of multiple simultaneous dumps to one host alleviates
1127 * the biggest&smallest dumps problem: both can be started at the
1131 diskp_accept = NULL;
1132 holdp_accept = NULL;
1133 delayed_diskp = NULL;
1135 cur_idle = NOT_IDLE;
1137 dumporder = getconf_str(CNF_DUMPORDER);
1138 if(strlen(dumporder) > (size_t)(dumper-dmptable)) {
1139 dumptype = dumporder[dumper-dmptable];
1142 if(dumper-dmptable < 3)
1150 if (!empty(directq)) {
1151 taper = idle_taper();
1153 TapeAction result_tape_action;
1154 char *why_no_new_tape = NULL;
1155 result_tape_action = tape_action(taper, &why_no_new_tape);
1156 if (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
1157 result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1158 off_t extra_tapes_size = 0;
1161 if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1162 extra_tapes_size = tape_length *
1163 (off_t)(conf_runtapes - current_tape);
1164 for (taper1 = tapetable;
1165 taper1 < tapetable + conf_taper_parallel_write;
1167 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
1168 extra_tapes_size += taper1->left;
1172 extra_tapes_size -= (sched(dp)->est_size -
1178 for (diskp = directq.head; diskp != NULL;
1179 diskp = diskp->next) {
1180 allow_dump_dle(diskp, taper, dumptype, &directq, now,
1181 dumper_to_holding, &cur_idle,
1182 &delayed_diskp, &diskp_accept,
1183 &holdp_accept, extra_tapes_size);
1186 diskp = diskp_accept;
1187 holdp = holdp_accept;
1197 if (diskp == NULL) {
1198 for(diskp = rq->head; diskp != NULL; diskp = diskp->next) {
1199 assert(diskp->host != NULL && sched(diskp) != NULL);
1201 allow_dump_dle(diskp, NULL, dumptype, rq, now,
1202 dumper_to_holding, &cur_idle, &delayed_diskp,
1203 &diskp_accept, &holdp_accept, 0);
1205 diskp = diskp_accept;
1206 holdp = holdp_accept;
1209 idle_reason = max(idle_reason, cur_idle);
1210 if (diskp == NULL && idle_reason == IDLE_NO_DISKSPACE) {
1211 /* continue flush waiting for new tape */
1216 * If we have no disk at this point, and there are disks that
1217 * are delayed, then schedule a time event to call this dumper
1218 * with the disk with the shortest delay.
1220 if (diskp == NULL && delayed_diskp != NULL) {
1221 assert(sleep_time > now);
1223 dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME,
1224 handle_dumpers_time, &runq);
1226 } else if (diskp != NULL && taper == NULL) {
1227 sched(diskp)->act_size = (off_t)0;
1228 allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1229 sched(diskp)->activehd = assign_holdingdisk(holdp, diskp);
1231 sched(diskp)->destname = newstralloc(sched(diskp)->destname,
1232 sched(diskp)->holdp[0]->destname);
1233 diskp->host->inprogress++; /* host is now busy */
1234 diskp->inprogress = 1;
1235 sched(diskp)->dumper = dumper;
1236 sched(diskp)->timestamp = now;
1237 amfree(diskp->dataport_list);
1239 dumper->busy = 1; /* dumper is now busy */
1240 dumper->dp = diskp; /* link disk to dumper */
1241 remove_disk(rq, diskp); /* take it off the run queue */
1243 sched(diskp)->origsize = (off_t)-1;
1244 sched(diskp)->dumpsize = (off_t)-1;
1245 sched(diskp)->dumptime = (time_t)0;
1246 sched(diskp)->tapetime = (time_t)0;
1247 chunker = dumper->chunker = &chktable[dumper - dmptable];
1248 chunker->result = LAST_TOK;
1249 dumper->result = LAST_TOK;
1250 startup_chunk_process(chunker,chunker_program);
1251 chunker_cmd(chunker, START, NULL, driver_timestamp);
1252 chunker->dumper = dumper;
1253 chunker_cmd(chunker, PORT_WRITE, diskp, NULL);
1254 cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
1256 assignedhd_t **h=NULL;
1258 char *qname = quote_string(diskp->name);
1260 g_printf(_("driver: did not get PORT from %s for %s:%s\n"),
1261 chunker->name, diskp->host->hostname, qname);
1265 deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1266 h = sched(diskp)->holdp;
1267 activehd = sched(diskp)->activehd;
1268 h[activehd]->used = 0;
1269 h[activehd]->disk->allocated_dumpers--;
1270 adjust_diskspace(diskp, DONE);
1271 delete_diskspace(diskp);
1272 diskp->host->inprogress--;
1273 diskp->inprogress = 0;
1274 sched(diskp)->dumper = NULL;
1277 sched(diskp)->dump_attempted++;
1278 free_serial_dp(diskp);
1279 if(sched(diskp)->dump_attempted < 2)
1280 enqueue_disk(rq, diskp);
1283 dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD,
1284 handle_dumper_result, dumper);
1285 chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD,
1286 handle_chunker_result, chunker);
1287 dumper->output_port = atoi(result_argv[1]);
1288 amfree(diskp->dataport_list);
1289 diskp->dataport_list = stralloc(result_argv[2]);
1291 if (diskp->host->pre_script == 0) {
1292 for (dp=diskp->host->disks; dp != NULL; dp = dp->hostnext) {
1293 run_server_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1294 get_config_name(), dp, -1);
1296 diskp->host->pre_script = 1;
1298 run_server_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1299 get_config_name(), diskp,
1300 sched(diskp)->level);
1301 dumper_cmd(dumper, PORT_DUMP, diskp, NULL);
1303 diskp->host->start_t = now + 15;
1304 if (empty(*rq)) force_flush = 1;
1307 g_strfreev(result_argv);
1309 } else if (diskp != NULL && taper != NULL) { /* dump to tape */
1310 sched(diskp)->act_size = (off_t)0;
1311 allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1312 diskp->host->inprogress++; /* host is now busy */
1313 diskp->inprogress = 1;
1314 sched(diskp)->dumper = dumper;
1315 sched(diskp)->taper = taper;
1316 sched(diskp)->timestamp = now;
1317 dumper->chunker = NULL;
1318 amfree(diskp->dataport_list);
1320 dumper->busy = 1; /* dumper is now busy */
1321 dumper->dp = diskp; /* link disk to dumper */
1322 remove_disk(&directq, diskp); /* take it off the direct queue */
1324 sched(diskp)->origsize = (off_t)-1;
1325 sched(diskp)->dumpsize = (off_t)-1;
1326 sched(diskp)->dumptime = (time_t)0;
1327 sched(diskp)->tapetime = (time_t)0;
1328 dumper->result = LAST_TOK;
1329 taper->result = LAST_TOK;
1330 taper->input_error = NULL;
1331 taper->tape_error = NULL;
1332 taper->disk = diskp;
1333 taper->first_label = NULL;
1335 taper->dumper = dumper;
1336 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1337 taper->state &= ~TAPER_STATE_IDLE;
1338 if (taper_nb_wait_reply == 0) {
1339 taper_ev_read = event_register(taper_fd, EV_READFD,
1340 handle_taper_result, NULL);
1343 taper_nb_wait_reply++;
1344 taper_cmd(PORT_WRITE, diskp, NULL, sched(diskp)->level,
1345 sched(diskp)->datestamp);
1346 diskp->host->start_t = now + 15;
1354 * This gets called when a dumper is delayed for some reason. It may
1355 * be because a disk has a delayed start, or amanda is constrained
1356 * by network or disk limits.
1360 handle_dumpers_time(
1363 disklist_t *runq = cookie;
1364 event_release(dumpers_ev_time);
1365 dumpers_ev_time = NULL;
1366 start_some_dumps(runq);
1377 g_printf(_("dump of driver schedule %s:\n--------\n"), str);
1379 for(dp = qp->head; dp != NULL; dp = dp->next) {
1380 qname = quote_string(dp->name);
1381 g_printf(" %-20s %-25s lv %d t %5lu s %lld p %d\n",
1382 dp->host->hostname, qname, sched(dp)->level,
1383 sched(dp)->est_time,
1384 (long long)sched(dp)->est_size, sched(dp)->priority);
1387 g_printf("--------\n");
1391 start_degraded_mode(
1392 /*@keep@*/ disklist_t *queuep)
1396 off_t est_full_size;
1399 newq.head = newq.tail = 0;
1401 dump_schedule(queuep, _("before start degraded mode"));
1403 est_full_size = (off_t)0;
1404 while(!empty(*queuep)) {
1405 dp = dequeue_disk(queuep);
1407 qname = quote_string(dp->name);
1408 if(sched(dp)->level != 0)
1409 /* go ahead and do the disk as-is */
1410 enqueue_disk(&newq, dp);
1412 if (reserved_space + est_full_size + sched(dp)->est_size
1413 <= total_disksize) {
1414 enqueue_disk(&newq, dp);
1415 est_full_size += sched(dp)->est_size;
1417 else if(sched(dp)->degr_level != -1) {
1418 sched(dp)->level = sched(dp)->degr_level;
1419 sched(dp)->dumpdate = sched(dp)->degr_dumpdate;
1420 sched(dp)->est_nsize = sched(dp)->degr_nsize;
1421 sched(dp)->est_csize = sched(dp)->degr_csize;
1422 sched(dp)->est_time = sched(dp)->degr_time;
1423 sched(dp)->est_kps = sched(dp)->degr_kps;
1424 enqueue_disk(&newq, dp);
1427 log_add(L_FAIL, "%s %s %s %d [%s]",
1428 dp->host->hostname, qname, sched(dp)->datestamp,
1429 sched(dp)->level, sched(dp)->degr_mesg);
1435 /*@i@*/ *queuep = newq;
1438 dump_schedule(queuep, _("after start degraded mode"));
1443 continue_port_dumps(void)
1447 int active_dumpers=0, busy_dumpers=0, i;
1450 /* First we try to grant diskspace to some dumps waiting for it. */
1451 for( dp = roomq.head; dp; dp = ndp ) {
1453 /* find last holdingdisk used by this dump */
1454 for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) {
1455 (void)h; /* Quiet lint */
1457 /* find more space */
1458 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1459 &active_dumpers, h[i] );
1461 for(dumper = dmptable; dumper < dmptable + inparallel &&
1462 dumper->dp != dp; dumper++) {
1463 (void)dp; /* Quiet lint */
1465 assert( dumper < dmptable + inparallel );
1466 sched(dp)->activehd = assign_holdingdisk( h, dp );
1467 chunker_cmd( dumper->chunker, CONTINUE, dp, NULL );
1469 remove_disk( &roomq, dp );
1473 /* So for some disks there is less holding diskspace available than
1474 * was asked for. Possible reasons are
1475 * a) diskspace has been allocated for other dumps which are
1476 * still running or already being written to tape
1477 * b) all other dumps have been suspended due to lack of diskspace
1478 * Case a) is not a problem. We just wait for the diskspace to
1479 * be freed by moving the current disk to a queue.
1480 * If case b) occurs, we have a deadlock situation. We select
1481 * a dump from the queue to be aborted and abort it. It will
1482 * be retried directly to tape.
1484 for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1485 if( dumper->busy ) {
1487 if( !find_disk(&roomq, dumper->dp) ) {
1490 sched(dp)->est_size > sched(dumper->dp)->est_size ) {
1495 if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) &&
1496 ((all_taper_idle() && empty(tapeq)) || degraded_mode) &&
1497 pending_aborts == 0 ) { /* case b */
1498 sched(dp)->no_space = 1;
1499 /* At this time, dp points to the dump with the smallest est_size.
1500 * We abort that dump, hopefully not wasting too much time retrying it.
1502 remove_disk( &roomq, dp );
1503 chunker_cmd(sched(dp)->dumper->chunker, ABORT, NULL, _("Not enough holding disk space"));
1504 dumper_cmd( sched(dp)->dumper, ABORT, NULL, _("Not enough holding disk space"));
1511 handle_taper_result(
1512 void *cookie G_GNUC_UNUSED)
1514 disk_t *dp = NULL, *dp1;
1521 taper_t *taper = NULL;
1526 assert(cookie == NULL);
1533 cmd = getresult(taper_fd, 1, &result_argc, &result_argv);
1538 if(result_argc != 2) {
1539 error(_("error: [taper FAILED result_argc != 2: %d"), result_argc);
1545 for (i=0; i < conf_taper_parallel_write; i++) {
1546 if (strcmp(tapetable[i].name, result_argv[1]) == 0) {
1547 taper= &tapetable[i];
1550 assert(taper != NULL);
1552 taper->state &= ~TAPER_STATE_INIT;
1553 taper->state |= TAPER_STATE_RESERVATION;
1554 taper->state |= TAPER_STATE_IDLE;
1555 amfree(taper->first_label);
1556 taper_nb_wait_reply--;
1557 taper_nb_scan_volume--;
1558 last_started_taper = taper;
1559 if (taper_nb_wait_reply == 0) {
1560 event_release(taper_ev_read);
1561 taper_ev_read = NULL;
1563 start_some_dumps(&runq);
1567 case FAILED: /* FAILED <handle> INPUT-* TAPE-* <input err mesg> <tape err mesg> */
1568 if(result_argc != 6) {
1569 error(_("error: [taper FAILED result_argc != 6: %d"), result_argc);
1573 dp = serial2disk(result_argv[1]);
1574 taper = sched(dp)->taper;
1575 assert(dp == taper->disk);
1577 free_serial(result_argv[1]);
1579 qname = quote_string(dp->name);
1580 g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1581 walltime_str(curclock()), dp->host->hostname, qname);
1584 if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1585 taper->input_error = newstralloc(taper->input_error, result_argv[4]);
1586 taper->result = FAILED;
1589 } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1590 taper->tape_error = newstralloc(taper->tape_error,
1591 _("Taper protocol error"));
1592 taper->result = FAILED;
1593 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1594 dp->host->hostname, qname, sched(dp)->datestamp,
1595 sched(dp)->level, taper->tape_error);
1599 if (strcmp(result_argv[3], "TAPE-ERROR") == 0) {
1600 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1601 taper->tape_error = newstralloc(taper->tape_error, result_argv[5]);
1602 taper->result = FAILED;
1605 } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1606 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1607 taper->tape_error = newstralloc(taper->tape_error,
1608 _("Taper protocol error"));
1609 taper->result = FAILED;
1610 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1611 dp->host->hostname, qname, sched(dp)->datestamp,
1612 sched(dp)->level, taper->tape_error);
1618 taper->result = cmd;
1622 case PARTIAL: /* PARTIAL <handle> INPUT-* TAPE-* <stat mess> <input err mesg> <tape err mesg>*/
1623 case DONE: /* DONE <handle> INPUT-GOOD TAPE-GOOD <stat mess> <input err mesg> <tape err mesg> */
1624 if(result_argc != 7) {
1625 error(_("error: [taper PARTIAL result_argc != 7: %d"), result_argc);
1629 dp = serial2disk(result_argv[1]);
1630 taper = sched(dp)->taper;
1631 assert(dp == taper->disk);
1633 free_serial(result_argv[1]);
1635 qname = quote_string(dp->name);
1636 g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1637 walltime_str(curclock()), dp->host->hostname, qname);
1640 if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1641 taper->input_error = newstralloc(taper->input_error, result_argv[5]);
1642 taper->result = FAILED;
1645 } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1646 taper->tape_error = newstralloc(taper->tape_error,
1647 _("Taper protocol error"));
1648 taper->result = FAILED;
1649 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1650 dp->host->hostname, qname, sched(dp)->datestamp,
1651 sched(dp)->level, taper->tape_error);
1655 if (strcmp(result_argv[3], "TAPE-ERROR") == 0) {
1656 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1657 taper->tape_error = newstralloc(taper->tape_error, result_argv[6]);
1658 taper->result = FAILED;
1661 } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1662 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1663 taper->tape_error = newstralloc(taper->tape_error,
1664 _("Taper protocol error"));
1665 taper->result = FAILED;
1666 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1667 dp->host->hostname, qname, sched(dp)->datestamp,
1668 sched(dp)->level, taper->tape_error);
1673 s = strstr(result_argv[4], " kb ");
1676 sched(dp)->dumpsize = atol(s);
1679 taper->result = cmd;
1684 case PARTDONE: /* PARTDONE <handle> <label> <fileno> <kbytes> <stat> */
1685 dp = serial2disk(result_argv[1]);
1686 taper = sched(dp)->taper;
1687 assert(dp == taper->disk);
1688 if (result_argc != 6) {
1689 error(_("error [taper PARTDONE result_argc != 6: %d]"),
1693 if (!taper->first_label) {
1694 amfree(taper->first_label);
1695 taper->first_label = stralloc(result_argv[2]);
1696 taper->first_fileno = OFF_T_ATOI(result_argv[3]);
1698 taper->written += OFF_T_ATOI(result_argv[4]);
1699 if (taper->written > sched(taper->disk)->act_size)
1700 sched(taper->disk)->act_size = taper->written;
1703 s = strstr(result_argv[5], " kb ");
1708 taper->left -= partsize;
1712 case REQUEST_NEW_TAPE: /* REQUEST-NEW-TAPE <handle> */
1713 if (result_argc != 2) {
1714 error(_("error [taper REQUEST_NEW_TAPE result_argc != 2: %d]"),
1719 dp = serial2disk(result_argv[1]);
1720 taper = sched(dp)->taper;
1721 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1722 taper->state |= TAPER_STATE_TAPE_REQUESTED;
1724 start_some_dumps(&runq);
1728 case NEW_TAPE: /* NEW-TAPE <handle> <label> */
1729 if (result_argc != 3) {
1730 error(_("error [taper NEW_TAPE result_argc != 3: %d]"),
1736 taper_nb_scan_volume--;
1737 dp = serial2disk(result_argv[1]);
1738 taper = sched(dp)->taper;
1739 /* Update our tape counter and reset taper->left */
1741 taper->left = tape_length;
1742 taper->state &= ~TAPER_STATE_WAIT_NEW_TAPE;
1743 taper->state |= TAPER_STATE_TAPE_STARTED;
1744 last_started_taper = NULL;
1746 /* start a new worker */
1747 for (i = 0; i < conf_taper_parallel_write ; i++) {
1748 taper1 = &tapetable[i];
1749 if (need_degraded == 0 &&
1750 taper1->state == TAPER_STATE_DEFAULT) {
1751 taper1->state = TAPER_STATE_INIT;
1752 if (taper_nb_wait_reply == 0) {
1753 taper_ev_read = event_register(taper_fd, EV_READFD,
1754 handle_taper_result, NULL);
1756 taper_nb_wait_reply++;
1757 taper_nb_scan_volume++;
1758 taper_cmd(START_TAPER, NULL, taper1->name, 0,
1765 case NO_NEW_TAPE: /* NO-NEW-TAPE <handle> */
1766 if (result_argc != 2) {
1767 error(_("error [taper NO_NEW_TAPE result_argc != 2: %d]"),
1772 taper_nb_scan_volume--;
1773 dp = serial2disk(result_argv[1]);
1774 taper = sched(dp)->taper;
1775 taper->state |= TAPER_STATE_DONE;
1776 last_started_taper = NULL;
1777 start_degraded_mode(&runq);
1780 case DUMPER_STATUS: /* DUMPER-STATUS <handle> */
1781 if (result_argc != 2) {
1782 error(_("error [taper DUMPER_STATUS result_argc != 2: %d]"),
1786 dp = serial2disk(result_argv[1]);
1787 taper = sched(dp)->taper;
1788 if (taper->dumper->result == LAST_TOK) {
1789 taper->sendresult = 1;
1791 if( taper->dumper->result == DONE) {
1792 taper_cmd(DONE, dp, NULL, 0, NULL);
1794 taper_cmd(FAILED, dp, NULL, 0, NULL);
1799 case TAPE_ERROR: /* TAPE-ERROR <name> <err mess> */
1801 if (strcmp(result_argv[1], "SETUP") == 0) {
1802 taper_nb_wait_reply = 0;
1803 taper_nb_scan_volume = 0;
1805 taper = taper_from_name(result_argv[1]);
1806 taper->state = TAPER_STATE_DONE;
1808 q = quote_string(result_argv[2]);
1809 log_add(L_WARNING, _("Taper error: %s"), q);
1812 taper->tape_error = newstralloc(taper->tape_error,
1816 taper_nb_wait_reply--;
1817 taper_nb_scan_volume--;
1819 if (taper_nb_wait_reply == 0) {
1820 event_release(taper_ev_read);
1821 taper_ev_read = NULL;
1824 if (schedule_done && !degraded_mode) {
1825 start_degraded_mode(&runq);
1827 start_some_dumps(&runq);
1830 case PORT: /* PORT <name> <handle> <port> <dataport_list> */
1831 dp = serial2disk(result_argv[2]);
1832 taper = sched(dp)->taper;
1833 dumper = sched(dp)->dumper;
1834 dumper->output_port = atoi(result_argv[3]);
1835 amfree(dp->dataport_list);
1836 dp->dataport_list = stralloc(result_argv[4]);
1838 amfree(taper->input_error);
1839 amfree(taper->tape_error);
1840 amfree(taper->first_label);
1842 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1844 if (dp->host->pre_script == 0) {
1845 for (dp1=dp->host->disks; dp1 != NULL; dp1 = dp1->hostnext) {
1846 run_server_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1847 get_config_name(), dp1, -1);
1849 dp->host->pre_script = 1;
1851 run_server_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1852 get_config_name(), dp,
1854 /* tell the dumper to dump to a port */
1855 dumper_cmd(dumper, PORT_DUMP, dp, NULL);
1856 dp->host->start_t = time(NULL) + 15;
1857 amfree(dp->dataport_list);
1859 taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1861 dumper->ev_read = event_register(dumper->fd, EV_READFD,
1862 handle_dumper_result, dumper);
1866 log_add(L_WARNING, _("Taper protocol error"));
1868 * Since we received a taper error, we can't send anything more
1869 * to the taper. Go into degraded mode to try to get everthing
1870 * onto disk. Later, these dumps can be flushed to a new tape.
1871 * The tape queue is zapped so that it appears empty in future
1872 * checks. If there are dumps waiting for diskspace to be freed,
1877 _("going into degraded mode because of taper component error."));
1880 for (taper = tapetable;
1881 taper < tapetable + conf_taper_parallel_write;
1883 if (taper && taper->disk && taper->result != LAST_TOK) {
1884 taper->tape_error = newstralloc(taper->tape_error,"BOGUS");
1885 taper->result = cmd;
1886 if (taper->dumper) {
1887 if (taper->dumper->result != LAST_TOK) {
1888 // Dumper already returned it's result
1889 dumper_taper_result(taper->disk);
1892 file_taper_result(taper->disk);
1899 if(taper_ev_read != NULL) {
1900 event_release(taper_ev_read);
1901 taper_ev_read = NULL;
1902 taper_nb_wait_reply = 0;
1904 start_degraded_mode(&runq);
1905 tapeq.head = tapeq.tail = NULL;
1911 error(_("driver received unexpected token (%s) from taper"),
1916 g_strfreev(result_argv);
1918 if (taper && taper->disk && taper->result != LAST_TOK) {
1920 if (taper->dumper->result != LAST_TOK) {
1921 // Dumper already returned it's result
1922 dumper_taper_result(taper->disk);
1925 file_taper_result(taper->disk);
1929 } while(areads_dataready(taper_fd));
1939 char *qname = quote_string(dp->name);
1941 taper = sched(dp)->taper;
1942 if (taper->result == DONE) {
1943 update_info_taper(dp, taper->first_label, taper->first_fileno,
1947 sched(dp)->taper_attempted += 1;
1949 if (taper->input_error) {
1950 g_printf("driver: taper failed %s %s: %s\n",
1951 dp->host->hostname, qname, taper->input_error);
1952 if (strcmp(sched(dp)->datestamp, driver_timestamp) == 0) {
1953 if(sched(dp)->taper_attempted >= 2) {
1954 log_add(L_FAIL, _("%s %s %s %d [too many taper retries after holding disk error: %s]"),
1955 dp->host->hostname, qname, sched(dp)->datestamp,
1956 sched(dp)->level, taper->input_error);
1957 g_printf("driver: taper failed %s %s, too many taper retry after holding disk error\n",
1958 dp->host->hostname, qname);
1959 amfree(sched(dp)->destname);
1960 amfree(sched(dp)->dumpdate);
1961 amfree(sched(dp)->degr_dumpdate);
1962 amfree(sched(dp)->degr_mesg);
1963 amfree(sched(dp)->datestamp);
1966 log_add(L_INFO, _("%s %s %s %d [Will retry dump because of holding disk error: %s]"),
1967 dp->host->hostname, qname, sched(dp)->datestamp,
1968 sched(dp)->level, taper->input_error);
1969 g_printf("driver: taper will retry %s %s because of holding disk error\n",
1970 dp->host->hostname, qname);
1971 if (dp->to_holdingdisk != HOLD_REQUIRED) {
1972 dp->to_holdingdisk = HOLD_NEVER;
1973 sched(dp)->dump_attempted -= 1;
1974 headqueue_disk(&directq, dp);
1976 amfree(sched(dp)->destname);
1977 amfree(sched(dp)->dumpdate);
1978 amfree(sched(dp)->degr_dumpdate);
1979 amfree(sched(dp)->degr_mesg);
1980 amfree(sched(dp)->datestamp);
1985 amfree(sched(dp)->destname);
1986 amfree(sched(dp)->dumpdate);
1987 amfree(sched(dp)->degr_dumpdate);
1988 amfree(sched(dp)->degr_mesg);
1989 amfree(sched(dp)->datestamp);
1992 } else if (taper->tape_error) {
1993 g_printf("driver: taper failed %s %s with tape error: %s\n",
1994 dp->host->hostname, qname, taper->tape_error);
1995 if(sched(dp)->taper_attempted >= 2) {
1996 log_add(L_FAIL, _("%s %s %s %d [too many taper retries]"),
1997 dp->host->hostname, qname, sched(dp)->datestamp,
1999 g_printf("driver: taper failed %s %s, too many taper retry\n",
2000 dp->host->hostname, qname);
2001 amfree(sched(dp)->destname);
2002 amfree(sched(dp)->dumpdate);
2003 amfree(sched(dp)->degr_dumpdate);
2004 amfree(sched(dp)->degr_mesg);
2005 amfree(sched(dp)->datestamp);
2008 g_printf("driver: taper will retry %s %s\n",
2009 dp->host->hostname, qname);
2010 /* Re-insert into taper queue. */
2011 headqueue_disk(&tapeq, dp);
2013 } else if (taper->result != DONE) {
2014 g_printf("driver: taper failed %s %s without error\n",
2015 dp->host->hostname, qname);
2017 delete_diskspace(dp);
2018 amfree(sched(dp)->destname);
2019 amfree(sched(dp)->dumpdate);
2020 amfree(sched(dp)->degr_dumpdate);
2021 amfree(sched(dp)->degr_mesg);
2022 amfree(sched(dp)->datestamp);
2028 taper->state &= ~TAPER_STATE_FILE_TO_TAPE;
2029 taper->state |= TAPER_STATE_IDLE;
2030 amfree(taper->input_error);
2031 amfree(taper->tape_error);
2033 taper_nb_wait_reply--;
2034 if (taper_nb_wait_reply == 0) {
2035 event_release(taper_ev_read);
2036 taper_ev_read = NULL;
2039 /* continue with those dumps waiting for diskspace */
2040 continue_port_dumps();
2041 start_some_dumps(&runq);
2046 dumper_taper_result(
2054 dumper = sched(dp)->dumper;
2055 taper = sched(dp)->taper;
2058 if(dumper->result == DONE && taper->result == DONE) {
2059 update_info_dumper(dp, sched(dp)->origsize,
2060 sched(dp)->dumpsize, sched(dp)->dumptime);
2061 update_info_taper(dp, taper->first_label, taper->first_fileno,
2063 qname = quote_string(dp->name); /*quote to take care of spaces*/
2065 log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2066 dp->host->hostname, qname, sched(dp)->datestamp,
2068 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2069 (long long)sched(dp)->est_csize,
2070 sched(dp)->est_kps);
2073 update_failed_dump(dp);
2076 is_partial = dumper->result != DONE || taper->result != DONE;
2078 sched(dp)->dump_attempted += 1;
2079 sched(dp)->taper_attempted += 1;
2081 if((dumper->result != DONE || taper->result != DONE) &&
2082 sched(dp)->dump_attempted <= 1 &&
2083 sched(dp)->taper_attempted <= 1) {
2084 enqueue_disk(&directq, dp);
2087 if(dumper->ev_read != NULL) {
2088 event_release(dumper->ev_read);
2089 dumper->ev_read = NULL;
2091 taper_nb_wait_reply--;
2092 if (taper_nb_wait_reply == 0 && taper_ev_read != NULL) {
2093 event_release(taper_ev_read);
2094 taper_ev_read = NULL;
2096 taper->state &= ~TAPER_STATE_DUMP_TO_TAPE;
2097 taper->state |= TAPER_STATE_IDLE;
2098 amfree(taper->input_error);
2099 amfree(taper->tape_error);
2101 dp->host->inprogress -= 1;
2103 deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2104 taper->dumper = NULL;
2106 sched(dp)->dumper = NULL;
2107 sched(dp)->taper = NULL;
2108 start_some_dumps(&runq);
2117 /* Use an already started taper first */
2118 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2120 if ((taper->state & TAPER_STATE_IDLE) &&
2121 (taper->state & TAPER_STATE_TAPE_STARTED) &&
2122 !(taper->state & TAPER_STATE_DONE) &&
2123 !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2124 !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2127 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2129 if ((taper->state & TAPER_STATE_IDLE) &&
2130 (taper->state & TAPER_STATE_RESERVATION) &&
2131 !(taper->state & TAPER_STATE_DONE) &&
2132 !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2133 !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2145 for (taper = tapetable; taper < tapetable+conf_taper_parallel_write;
2147 if (strcmp(taper->name, name) == 0) return taper;
2153 dumper_chunker_result(
2158 assignedhd_t **h=NULL;
2165 dumper = sched(dp)->dumper;
2166 chunker = dumper->chunker;
2170 h = sched(dp)->holdp;
2171 activehd = sched(dp)->activehd;
2173 if(dumper->result == DONE && chunker->result == DONE) {
2174 update_info_dumper(dp, sched(dp)->origsize,
2175 sched(dp)->dumpsize, sched(dp)->dumptime);
2176 qname = quote_string(dp->name);/*quote to take care of spaces*/
2178 log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2179 dp->host->hostname, qname, sched(dp)->datestamp,
2181 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2182 (long long)sched(dp)->est_csize,
2183 sched(dp)->est_kps);
2186 update_failed_dump(dp);
2189 deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2191 is_partial = dumper->result != DONE || chunker->result != DONE;
2192 rename_tmp_holding(sched(dp)->destname, !is_partial);
2193 holding_set_origsize(sched(dp)->destname, sched(dp)->origsize);
2196 for( i = 0, h = sched(dp)->holdp; i < activehd; i++ ) {
2197 dummy += h[i]->used;
2200 size = holding_file_size(sched(dp)->destname, 0);
2201 h[activehd]->used = size - dummy;
2202 h[activehd]->disk->allocated_dumpers--;
2203 adjust_diskspace(dp, DONE);
2205 sched(dp)->dump_attempted += 1;
2207 if((dumper->result != DONE || chunker->result != DONE) &&
2208 sched(dp)->dump_attempted <= 1) {
2209 delete_diskspace(dp);
2210 if (sched(dp)->no_space) {
2211 enqueue_disk(&directq, dp);
2213 enqueue_disk(&runq, dp);
2216 else if(size > (off_t)DISK_BLOCK_KB) {
2217 enqueue_disk(&tapeq, dp);
2220 delete_diskspace(dp);
2224 dp->host->inprogress -= 1;
2227 waitpid(chunker->pid, NULL, 0 );
2228 aclose(chunker->fd);
2233 if (chunker->result == ABORT_FINISHED)
2235 continue_port_dumps();
2237 * Wakeup any dumpers that are sleeping because of network
2238 * or disk constraints.
2240 start_some_dumps(&runq);
2246 handle_dumper_result(
2249 /* uses global pending_aborts */
2250 dumper_t *dumper = cookie;
2252 disk_t *dp, *sdp, *dp1;
2258 assert(dumper != NULL);
2261 assert(sched(dp) != NULL);
2266 cmd = getresult(dumper->fd, 1, &result_argc, &result_argv);
2269 /* result_argv[1] always contains the serial number */
2270 sdp = serial2disk(result_argv[1]);
2272 error(_("Invalid serial number %s"), result_argv[1]);
2273 g_assert_not_reached();
2277 qname = quote_string(dp->name);
2280 case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
2281 if(result_argc != 6) {
2282 error(_("error [dumper DONE result_argc != 6: %d]"), result_argc);
2286 sched(dp)->origsize = OFF_T_ATOI(result_argv[2]);
2287 sched(dp)->dumptime = TIME_T_ATOI(result_argv[4]);
2289 g_printf(_("driver: finished-cmd time %s %s dumped %s:%s\n"),
2290 walltime_str(curclock()), dumper->name,
2291 dp->host->hostname, qname);
2294 dumper->result = cmd;
2298 case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2300 * Requeue this disk, and fall through to the FAILED
2303 if(sched(dp)->dump_attempted) {
2304 char *qname = quote_string(dp->name);
2305 char *qerr = quote_string(result_argv[2]);
2306 log_add(L_FAIL, _("%s %s %s %d [too many dumper retry: %s]"),
2307 dp->host->hostname, qname, sched(dp)->datestamp,
2308 sched(dp)->level, qerr);
2309 g_printf(_("driver: dump failed %s %s %s, too many dumper retry: %s\n"),
2310 result_argv[1], dp->host->hostname, qname, qerr);
2315 case FAILED: /* FAILED <handle> <errstr> */
2316 /*free_serial(result_argv[1]);*/
2317 dumper->result = cmd;
2320 case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2322 * We sent an ABORT from the NO-ROOM case because this dump
2323 * wasn't going to fit onto the holding disk. We now need to
2324 * clean up the remains of this image, and try to finish
2325 * other dumps that are waiting on disk space.
2327 assert(pending_aborts);
2328 /*free_serial(result_argv[1]);*/
2329 dumper->result = cmd;
2333 /* either EOF or garbage from dumper. Turn it off */
2334 log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2335 dumper->name, (long)dumper->pid);
2336 if (dumper->ev_read) {
2337 event_release(dumper->ev_read);
2338 dumper->ev_read = NULL;
2342 dumper->down = 1; /* mark it down so it isn't used again */
2344 /* if it was dumping something, zap it and try again */
2345 if(sched(dp)->dump_attempted) {
2346 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2347 dp->host->hostname, qname, sched(dp)->datestamp,
2348 sched(dp)->level, dumper->name);
2350 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2351 dumper->name, dp->host->hostname, qname,
2354 dumper->result = cmd;
2361 g_strfreev(result_argv);
2367 run_server_scripts(EXECUTE_ON_POST_DLE_BACKUP,
2368 get_config_name(), dp, sched(dp)->level);
2369 /* check dump not yet started */
2370 for (dp1=runq.head; dp1 != NULL; dp1 = dp1->next) {
2371 if (dp1->host == dp->host)
2374 /* check direct to tape dump */
2375 for (dp1=directq.head; dp1 != NULL; dp1 = dp1->next) {
2376 if (dp1->host == dp->host)
2379 /* check dumping dle */
2380 for (dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
2381 if (dumper->busy && dumper->dp != dp &&
2382 dumper->dp->host == dp->host)
2385 if (last_dump && dp->host->post_script == 0) {
2386 if (dp->host->post_script == 0) {
2387 for (dp1=dp->host->disks; dp1 != NULL; dp1 = dp1->hostnext) {
2388 run_server_scripts(EXECUTE_ON_POST_HOST_BACKUP,
2389 get_config_name(), dp1, -1);
2391 dp->host->post_script = 1;
2396 taper = sched(dp)->taper;
2397 /* send the dumper result to the chunker */
2398 if (dumper->chunker) {
2399 if (dumper->chunker->down == 0 && dumper->chunker->fd != -1 &&
2400 dumper->chunker->result == LAST_TOK) {
2402 chunker_cmd(dumper->chunker, DONE, dp, NULL);
2405 chunker_cmd(dumper->chunker, FAILED, dp, NULL);
2408 if( dumper->result != LAST_TOK &&
2409 dumper->chunker->result != LAST_TOK)
2410 dumper_chunker_result(dp);
2411 } else { /* send the dumper result to the taper */
2412 if (taper->sendresult) {
2414 taper_cmd(DONE, dp, NULL, 0, NULL);
2416 taper_cmd(FAILED, dp, NULL, 0, NULL);
2418 taper->sendresult = 0;
2420 if (taper->dumper && taper->result != LAST_TOK) {
2421 dumper_taper_result(dp);
2424 } while(areads_dataready(dumper->fd));
2429 handle_chunker_result(
2432 chunker_t *chunker = cookie;
2433 assignedhd_t **h=NULL;
2443 assert(chunker != NULL);
2444 dumper = chunker->dumper;
2445 assert(dumper != NULL);
2448 assert(sched(dp) != NULL);
2449 assert(sched(dp)->destname != NULL);
2450 assert(dp != NULL && sched(dp) != NULL && sched(dp)->destname);
2452 if(sched(dp)->holdp) {
2453 h = sched(dp)->holdp;
2454 activehd = sched(dp)->activehd;
2460 cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
2463 /* result_argv[1] always contains the serial number */
2464 sdp = serial2disk(result_argv[1]);
2466 error(_("Invalid serial number %s"), result_argv[1]);
2467 g_assert_not_reached();
2473 case PARTIAL: /* PARTIAL <handle> <dumpsize> <errstr> */
2474 case DONE: /* DONE <handle> <dumpsize> <errstr> */
2475 if(result_argc != 4) {
2476 error(_("error [chunker %s result_argc != 4: %d]"), cmdstr[cmd],
2480 /*free_serial(result_argv[1]);*/
2482 sched(dp)->dumpsize = (off_t)atof(result_argv[2]);
2484 qname = quote_string(dp->name);
2485 g_printf(_("driver: finished-cmd time %s %s chunked %s:%s\n"),
2486 walltime_str(curclock()), chunker->name,
2487 dp->host->hostname, qname);
2491 event_release(chunker->ev_read);
2493 chunker->result = cmd;
2497 case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2498 event_release(chunker->ev_read);
2500 chunker->result = cmd;
2503 case FAILED: /* FAILED <handle> <errstr> */
2504 /*free_serial(result_argv[1]);*/
2506 event_release(chunker->ev_read);
2508 chunker->result = cmd;
2512 case NO_ROOM: /* NO-ROOM <handle> <missing_size> */
2513 if (!h || activehd < 0) { /* should never happen */
2514 error(_("!h || activehd < 0"));
2517 h[activehd]->used -= OFF_T_ATOI(result_argv[2]);
2518 h[activehd]->reserved -= OFF_T_ATOI(result_argv[2]);
2519 h[activehd]->disk->allocated_space -= OFF_T_ATOI(result_argv[2]);
2520 h[activehd]->disk->disksize -= OFF_T_ATOI(result_argv[2]);
2523 case RQ_MORE_DISK: /* RQ-MORE-DISK <handle> */
2524 if (!h || activehd < 0) { /* should never happen */
2525 error(_("!h || activehd < 0"));
2528 h[activehd]->disk->allocated_dumpers--;
2529 h[activehd]->used = h[activehd]->reserved;
2530 if( h[++activehd] ) { /* There's still some allocated space left.
2531 * Tell the dumper about it. */
2532 sched(dp)->activehd++;
2533 chunker_cmd( chunker, CONTINUE, dp, NULL );
2534 } else { /* !h[++activehd] - must allocate more space */
2535 sched(dp)->act_size = sched(dp)->est_size; /* not quite true */
2536 sched(dp)->est_size = (sched(dp)->act_size/(off_t)20) * (off_t)21; /* +5% */
2537 sched(dp)->est_size = am_round(sched(dp)->est_size, (off_t)DISK_BLOCK_KB);
2538 if (sched(dp)->est_size < sched(dp)->act_size + 2*DISK_BLOCK_KB)
2539 sched(dp)->est_size += 2 * DISK_BLOCK_KB;
2540 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
2544 /* No diskspace available. The reason for this will be
2545 * determined in continue_port_dumps(). */
2546 enqueue_disk( &roomq, dp );
2547 continue_port_dumps();
2548 /* continue flush waiting for new tape */
2551 /* OK, allocate space for disk and have chunker continue */
2552 sched(dp)->activehd = assign_holdingdisk( h, dp );
2553 chunker_cmd( chunker, CONTINUE, dp, NULL );
2559 case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2561 * We sent an ABORT from the NO-ROOM case because this dump
2562 * wasn't going to fit onto the holding disk. We now need to
2563 * clean up the remains of this image, and try to finish
2564 * other dumps that are waiting on disk space.
2566 /*assert(pending_aborts);*/
2568 /*free_serial(result_argv[1]);*/
2570 event_release(chunker->ev_read);
2572 chunker->result = cmd;
2577 /* either EOF or garbage from chunker. Turn it off */
2578 log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2579 chunker->name, (long)chunker->pid);
2581 /* if it was dumping something, zap it and try again */
2582 g_assert(h && activehd >= 0);
2583 qname = quote_string(dp->name);
2584 if(sched(dp)->dump_attempted) {
2585 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2586 dp->host->hostname, qname, sched(dp)->datestamp,
2587 sched(dp)->level, chunker->name);
2589 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2590 chunker->name, dp->host->hostname, qname,
2596 event_release(chunker->ev_read);
2598 chunker->result = cmd;
2605 g_strfreev(result_argv);
2607 if(chunker->result != LAST_TOK && chunker->dumper->result != LAST_TOK)
2608 dumper_chunker_result(dp);
2610 } while(areads_dataready(chunker->fd));
2621 char *hostname, *diskname, *datestamp;
2625 char *inpline = NULL;
2630 char *qdestname = NULL;
2631 char *conf_infofile;
2633 (void)cookie; /* Quiet unused parameter warning */
2635 event_release(flush_ev_read);
2636 flush_ev_read = NULL;
2638 /* read schedule from stdin */
2639 conf_infofile = config_dir_relative(getconf_str(CNF_INFOFILE));
2640 if (open_infofile(conf_infofile)) {
2641 error(_("could not open info db \"%s\""), conf_infofile);
2644 amfree(conf_infofile);
2646 for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2650 if (inpline[0] == '\0')
2656 skip_whitespace(s, ch); /* find the command */
2658 error(_("flush line %d: syntax error (no command)"), line);
2662 skip_non_whitespace(s, ch);
2665 if(strcmp(command,"ENDFLUSH") == 0) {
2669 if(strcmp(command,"FLUSH") != 0) {
2670 error(_("flush line %d: syntax error (%s != FLUSH)"), line, command);
2674 skip_whitespace(s, ch); /* find the hostname */
2676 error(_("flush line %d: syntax error (no hostname)"), line);
2680 skip_non_whitespace(s, ch);
2683 skip_whitespace(s, ch); /* find the diskname */
2685 error(_("flush line %d: syntax error (no diskname)"), line);
2689 skip_quoted_string(s, ch);
2690 s[-1] = '\0'; /* terminate the disk name */
2691 diskname = unquote_string(qname);
2693 skip_whitespace(s, ch); /* find the datestamp */
2695 error(_("flush line %d: syntax error (no datestamp)"), line);
2699 skip_non_whitespace(s, ch);
2702 skip_whitespace(s, ch); /* find the level number */
2703 if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2704 error(_("flush line %d: syntax error (bad level)"), line);
2707 skip_integer(s, ch);
2709 skip_whitespace(s, ch); /* find the filename */
2711 error(_("flush line %d: syntax error (no filename)"), line);
2715 skip_quoted_string(s, ch);
2717 destname = unquote_string(qdestname);
2719 holding_file_get_dumpfile(destname, &file);
2720 if( file.type != F_DUMPFILE) {
2721 if( file.type != F_CONT_DUMPFILE )
2722 log_add(L_INFO, _("%s: ignoring cruft file."), destname);
2725 dumpfile_free_data(&file);
2729 if(strcmp(hostname, file.name) != 0 ||
2730 strcmp(diskname, file.disk) != 0 ||
2731 strcmp(datestamp, file.datestamp) != 0) {
2732 log_add(L_INFO, _("disk %s:%s not consistent with file %s"),
2733 hostname, diskname, destname);
2736 dumpfile_free_data(&file);
2741 dp = lookup_disk(file.name, file.disk);
2744 log_add(L_INFO, _("%s: disk %s:%s not in database, skipping it."),
2745 destname, file.name, file.disk);
2747 dumpfile_free_data(&file);
2751 if (file.dumplevel < 0 || file.dumplevel > 399) {
2752 log_add(L_INFO, _("%s: ignoring file with bogus dump level %d."),
2753 destname, file.dumplevel);
2755 dumpfile_free_data(&file);
2759 if (holding_file_size(destname,1) <= 0) {
2760 log_add(L_INFO, "%s: removing file with no data.", destname);
2761 holding_file_unlink(destname);
2763 dumpfile_free_data(&file);
2767 dp1 = (disk_t *)alloc(SIZEOF(disk_t));
2769 dp1->next = dp1->prev = NULL;
2771 /* add it to the flushhost list */
2773 flushhost = alloc(SIZEOF(am_host_t));
2774 flushhost->next = NULL;
2775 flushhost->hostname = stralloc("FLUSHHOST");
2776 flushhost->up = NULL;
2777 flushhost->features = NULL;
2779 dp1->hostnext = flushhost->disks;
2780 flushhost->disks = dp1;
2782 sp = (sched_t *) alloc(SIZEOF(sched_t));
2783 sp->destname = destname;
2784 sp->level = file.dumplevel;
2785 sp->dumpdate = NULL;
2786 sp->degr_dumpdate = NULL;
2787 sp->degr_mesg = NULL;
2788 sp->datestamp = stralloc(file.datestamp);
2789 sp->est_nsize = (off_t)0;
2790 sp->est_csize = (off_t)0;
2793 sp->origsize = file.orig_size;
2795 sp->degr_level = -1;
2796 sp->dump_attempted = 0;
2797 sp->taper_attempted = 0;
2798 sp->act_size = holding_file_size(destname, 0);
2799 sp->holdp = build_diskspace(destname);
2800 if(sp->holdp == NULL) continue;
2803 sp->timestamp = (time_t)0;
2805 dp1->up = (char *)sp;
2807 enqueue_disk(&tapeq, dp1);
2808 dumpfile_free_data(&file);
2815 schedule_ev_read = event_register((event_id_t)0, EV_READFD,
2816 read_schedule, NULL);
2828 int level, line, priority;
2829 char *dumpdate, *degr_dumpdate, *degr_mesg;
2831 time_t time, degr_time;
2832 time_t *time_p = &time;
2833 time_t *degr_time_p = °r_time;
2834 off_t nsize, csize, degr_nsize, degr_csize;
2835 unsigned long kps, degr_kps;
2836 char *hostname, *features, *diskname, *datestamp, *inpline = NULL;
2840 off_t flush_size = (off_t)0;
2845 long long degr_nsize_;
2846 long long degr_csize_;
2847 GPtrArray *errarray;
2849 (void)cookie; /* Quiet unused parameter warning */
2851 event_release(schedule_ev_read);
2852 schedule_ev_read = NULL;
2854 /* read schedule from stdin */
2856 for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2857 if (inpline[0] == '\0')
2864 skip_whitespace(s, ch); /* find the command */
2866 error(_("schedule line %d: syntax error (no command)"), line);
2870 skip_non_whitespace(s, ch);
2873 if(strcmp(command,"DUMP") != 0) {
2874 error(_("schedule line %d: syntax error (%s != DUMP)"), line, command);
2878 skip_whitespace(s, ch); /* find the host name */
2880 error(_("schedule line %d: syntax error (no host name)"), line);
2884 skip_non_whitespace(s, ch);
2887 skip_whitespace(s, ch); /* find the feature list */
2889 error(_("schedule line %d: syntax error (no feature list)"), line);
2893 skip_non_whitespace(s, ch);
2896 skip_whitespace(s, ch); /* find the disk name */
2898 error(_("schedule line %d: syntax error (no disk name)"), line);
2902 skip_quoted_string(s, ch);
2903 s[-1] = '\0'; /* terminate the disk name */
2904 diskname = unquote_string(qname);
2906 skip_whitespace(s, ch); /* find the datestamp */
2908 error(_("schedule line %d: syntax error (no datestamp)"), line);
2912 skip_non_whitespace(s, ch);
2915 skip_whitespace(s, ch); /* find the priority number */
2916 if(ch == '\0' || sscanf(s - 1, "%d", &priority) != 1) {
2917 error(_("schedule line %d: syntax error (bad priority)"), line);
2920 skip_integer(s, ch);
2922 skip_whitespace(s, ch); /* find the level number */
2923 if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2924 error(_("schedule line %d: syntax error (bad level)"), line);
2927 skip_integer(s, ch);
2929 skip_whitespace(s, ch); /* find the dump date */
2931 error(_("schedule line %d: syntax error (bad dump date)"), line);
2935 skip_non_whitespace(s, ch);
2938 skip_whitespace(s, ch); /* find the native size */
2940 if(ch == '\0' || sscanf(s - 1, "%lld", &nsize_) != 1) {
2941 error(_("schedule line %d: syntax error (bad nsize)"), line);
2944 nsize = (off_t)nsize_;
2945 skip_integer(s, ch);
2947 skip_whitespace(s, ch); /* find the compressed size */
2949 if(ch == '\0' || sscanf(s - 1, "%lld", &csize_) != 1) {
2950 error(_("schedule line %d: syntax error (bad csize)"), line);
2953 csize = (off_t)csize_;
2954 skip_integer(s, ch);
2956 skip_whitespace(s, ch); /* find the time number */
2957 if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
2958 error(_("schedule line %d: syntax error (bad estimated time)"), line);
2961 *time_p = (time_t)time_;
2962 skip_integer(s, ch);
2964 skip_whitespace(s, ch); /* find the kps number */
2965 if(ch == '\0' || sscanf(s - 1, "%lu", &kps) != 1) {
2966 error(_("schedule line %d: syntax error (bad kps)"), line);
2969 skip_integer(s, ch);
2971 degr_dumpdate = NULL; /* flag if degr fields found */
2972 skip_whitespace(s, ch); /* find the degr level number */
2976 skip_quoted_string(s, ch);
2977 s[-1] = '\0'; /* terminate degr mesg */
2978 degr_mesg = unquote_string(qname);
2980 degr_nsize = (off_t)0;
2981 degr_csize = (off_t)0;
2982 degr_time = (time_t)0;
2984 } else if (ch != '\0') {
2985 if(sscanf(s - 1, "%d", °r_level) != 1) {
2986 error(_("schedule line %d: syntax error (bad degr level)"), line);
2989 skip_integer(s, ch);
2991 skip_whitespace(s, ch); /* find the degr dump date */
2993 error(_("schedule line %d: syntax error (bad degr dump date)"), line);
2996 degr_dumpdate = s - 1;
2997 skip_non_whitespace(s, ch);
3000 skip_whitespace(s, ch); /* find the degr native size */
3001 degr_nsize_ = (off_t)0;
3002 if(ch == '\0' || sscanf(s - 1, "%lld", °r_nsize_) != 1) {
3003 error(_("schedule line %d: syntax error (bad degr nsize)"), line);
3006 degr_nsize = (off_t)degr_nsize_;
3007 skip_integer(s, ch);
3009 skip_whitespace(s, ch); /* find the degr compressed size */
3010 degr_csize_ = (off_t)0;
3011 if(ch == '\0' || sscanf(s - 1, "%lld", °r_csize_) != 1) {
3012 error(_("schedule line %d: syntax error (bad degr csize)"), line);
3015 degr_csize = (off_t)degr_csize_;
3016 skip_integer(s, ch);
3018 skip_whitespace(s, ch); /* find the degr time number */
3019 if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
3020 error(_("schedule line %d: syntax error (bad degr estimated time)"), line);
3023 *degr_time_p = (time_t)time_;
3024 skip_integer(s, ch);
3026 skip_whitespace(s, ch); /* find the degr kps number */
3027 if(ch == '\0' || sscanf(s - 1, "%lu", °r_kps) != 1) {
3028 error(_("schedule line %d: syntax error (bad degr kps)"), line);
3031 skip_integer(s, ch);
3033 error(_("schedule line %d: no degraded estimate or message"), line);
3036 dp = lookup_disk(hostname, diskname);
3039 _("schedule line %d: %s:'%s' not in disklist, ignored"),
3040 line, hostname, qname);
3045 sp = (sched_t *) alloc(SIZEOF(sched_t));
3048 sp->dumpdate = stralloc(dumpdate);
3049 sp->est_nsize = DISK_BLOCK_KB + nsize; /* include header */
3050 sp->est_csize = DISK_BLOCK_KB + csize; /* include header */
3051 /* round estimate to next multiple of DISK_BLOCK_KB */
3052 sp->est_csize = am_round(sp->est_csize, DISK_BLOCK_KB);
3053 sp->est_size = sp->est_csize;
3054 sp->est_time = time;
3056 sp->priority = priority;
3057 sp->datestamp = stralloc(datestamp);
3060 sp->degr_level = degr_level;
3061 sp->degr_dumpdate = stralloc(degr_dumpdate);
3062 sp->degr_nsize = DISK_BLOCK_KB + degr_nsize;
3063 sp->degr_csize = DISK_BLOCK_KB + degr_csize;
3064 /* round estimate to next multiple of DISK_BLOCK_KB */
3065 sp->degr_csize = am_round(sp->degr_csize, DISK_BLOCK_KB);
3066 sp->degr_time = degr_time;
3067 sp->degr_kps = degr_kps;
3068 sp->degr_mesg = NULL;
3070 sp->degr_level = -1;
3071 sp->degr_dumpdate = NULL;
3072 sp->degr_mesg = degr_mesg;
3076 sp->dump_attempted = 0;
3077 sp->taper_attempted = 0;
3083 sp->timestamp = (time_t)0;
3084 sp->destname = NULL;
3087 dp->up = (char *) sp;
3088 if(dp->host->features == NULL) {
3089 dp->host->features = am_string_to_feature(features);
3090 if (!dp->host->features) {
3092 _("Invalid feature string from client '%s'"),
3094 dp->host->features = am_set_default_feature_set();
3097 remove_disk(&waitq, dp);
3099 errarray = validate_optionstr(dp);
3100 if (errarray->len > 0) {
3102 for (i=0; i < errarray->len; i++) {
3103 log_add(L_FAIL, _("%s %s %s 0 [%s]"),
3104 dp->host->hostname, qname,
3106 (char *)g_ptr_array_index(errarray, i));
3111 if (dp->data_path == DATA_PATH_DIRECTTCP &&
3112 dp->to_holdingdisk == HOLD_AUTO) {
3113 dp->to_holdingdisk = HOLD_NEVER;
3116 if (dp->to_holdingdisk == HOLD_NEVER) {
3117 enqueue_disk(&directq, dp);
3119 enqueue_disk(&runq, dp);
3121 flush_size += sp->act_size;
3125 g_printf(_("driver: flush size %lld\n"), (long long)flush_size);
3128 log_add(L_WARNING, _("WARNING: got empty schedule from planner"));
3129 if(need_degraded==1) start_degraded_mode(&runq);
3131 if (empty(runq)) force_flush = 1;
3132 start_some_dumps(&runq);
3136 static unsigned long
3144 unsigned long maxusage=0;
3145 unsigned long curusage=0;
3146 for(p = disklist_netifs(); p != NULL; p = p->next) {
3147 maxusage += interface_get_maxusage(p->config);
3148 curusage += p->curusage;
3150 if (maxusage >= curusage)
3151 res = maxusage - curusage;
3156 if ((unsigned long)interface_get_maxusage(ip->config) >= ip->curusage)
3157 res = interface_get_maxusage(ip->config) - ip->curusage;
3172 g_printf(_("driver: interface-state time %s"), time_str);
3174 for(ip = disklist_netifs(); ip != NULL; ip = ip->next) {
3175 g_printf(_(" if %s: free %lu"), interface_name(ip->config), free_kps(ip));
3185 ip->curusage += kps;
3189 deallocate_bandwidth(
3193 assert(kps <= ip->curusage);
3194 ip->curusage -= kps;
3205 total_free = (off_t)0;
3206 for(ha = holdalloc; ha != NULL; ha = ha->next) {
3207 diff = ha->disksize - ha->allocated_space;
3215 * We return an array of pointers to assignedhd_t. The array contains at
3216 * most one entry per holding disk. The list of pointers is terminated by
3217 * a NULL pointer. Each entry contains a pointer to a holdingdisk and
3218 * how much diskspace to use on that disk. Later on, assign_holdingdisk
3219 * will allocate the given amount of space.
3220 * If there is not enough room on the holdingdisks, NULL is returned.
3223 static assignedhd_t **
3227 assignedhd_t * pref)
3229 assignedhd_t **result = NULL;
3230 holdalloc_t *ha, *minp;
3234 off_t halloc, dalloc, hfree, dfree;
3236 (void)cur_idle; /* Quiet unused parameter warning */
3238 if (size < 2*DISK_BLOCK_KB)
3239 size = 2*DISK_BLOCK_KB;
3240 size = am_round(size, (off_t)DISK_BLOCK_KB);
3242 hold_debug(1, _("find_diskspace: want %lld K\n"),
3245 used = alloc(SIZEOF(*used) * num_holdalloc);/*disks used during this run*/
3246 memset( used, 0, (size_t)num_holdalloc );
3247 result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3250 while( i < num_holdalloc && size > (off_t)0 ) {
3251 /* find the holdingdisk with the fewest active dumpers and among
3252 * those the one with the biggest free space
3254 minp = NULL; minj = -1;
3255 for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3256 if( pref && pref->disk == ha && !used[j] &&
3257 ha->allocated_space <= ha->disksize - (off_t)DISK_BLOCK_KB) {
3262 else if( ha->allocated_space <= ha->disksize - (off_t)(2*DISK_BLOCK_KB) &&
3265 ha->allocated_dumpers < minp->allocated_dumpers ||
3266 (ha->allocated_dumpers == minp->allocated_dumpers &&
3267 ha->disksize-ha->allocated_space > minp->disksize-minp->allocated_space)) ) {
3274 if( !minp ) { break; } /* all holding disks are full */
3277 /* hfree = free space on the disk */
3278 hfree = minp->disksize - minp->allocated_space;
3280 /* dfree = free space for data, remove 1 header for each chunksize */
3281 dfree = hfree - (((hfree-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3283 /* dalloc = space I can allocate for data */
3284 dalloc = ( dfree < size ) ? dfree : size;
3286 /* halloc = space to allocate, including 1 header for each chunksize */
3287 halloc = dalloc + (((dalloc-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3289 hold_debug(1, _("find_diskspace: find diskspace: size %lld hf %lld df %lld da %lld ha %lld\n"),
3296 result[i] = alloc(SIZEOF(assignedhd_t));
3297 result[i]->disk = minp;
3298 result[i]->reserved = halloc;
3299 result[i]->used = (off_t)0;
3300 result[i]->destname = NULL;
3306 if(size != (off_t)0) { /* not enough space available */
3307 g_printf(_("find diskspace: not enough diskspace. Left with %lld K\n"), (long long)size);
3309 free_assignedhd(result);
3313 if (debug_holding > 1) {
3314 for( i = 0; result && result[i]; i++ ) {
3315 hold_debug(1, _("find_diskspace: find diskspace: selected %s free %lld reserved %lld dumpers %d\n"),
3316 holdingdisk_get_diskdir(result[i]->disk->hdisk),
3317 (long long)(result[i]->disk->disksize -
3318 result[i]->disk->allocated_space),
3319 (long long)result[i]->reserved,
3320 result[i]->disk->allocated_dumpers);
3329 assignedhd_t ** holdp,
3334 char *sfn = sanitise_filename(diskp->name);
3336 assignedhd_t **new_holdp;
3339 g_snprintf( lvl, SIZEOF(lvl), "%d", sched(diskp)->level );
3341 size = am_round(sched(diskp)->est_size - sched(diskp)->act_size,
3342 (off_t)DISK_BLOCK_KB);
3344 for( c = 0; holdp[c]; c++ )
3345 (void)c; /* count number of disks */
3347 /* allocate memory for sched(diskp)->holdp */
3348 for(j = 0; sched(diskp)->holdp && sched(diskp)->holdp[j]; j++)
3349 (void)j; /* Quiet lint */
3350 new_holdp = (assignedhd_t **)alloc(SIZEOF(assignedhd_t*)*(j+c+1));
3351 if (sched(diskp)->holdp) {
3352 memcpy(new_holdp, sched(diskp)->holdp, j * SIZEOF(*new_holdp));
3353 amfree(sched(diskp)->holdp);
3355 sched(diskp)->holdp = new_holdp;
3359 if( j > 0 ) { /* This is a request for additional diskspace. See if we can
3360 * merge assignedhd_t's */
3362 if( sched(diskp)->holdp[j-1]->disk == holdp[0]->disk ) { /* Yes! */
3363 sched(diskp)->holdp[j-1]->reserved += holdp[0]->reserved;
3364 holdp[0]->disk->allocated_space += holdp[0]->reserved;
3365 size = (holdp[0]->reserved>size) ? (off_t)0 : size-holdp[0]->reserved;
3366 qname = quote_string(diskp->name);
3367 hold_debug(1, _("assign_holdingdisk: merging holding disk %s to disk %s:%s, add %lld for reserved %lld, left %lld\n"),
3368 holdingdisk_get_diskdir(
3369 sched(diskp)->holdp[j-1]->disk->hdisk),
3370 diskp->host->hostname, qname,
3371 (long long)holdp[0]->reserved,
3372 (long long)sched(diskp)->holdp[j-1]->reserved,
3381 /* copy assignedhd_s to sched(diskp), adjust allocated_space */
3382 for( ; holdp[i]; i++ ) {
3383 holdp[i]->destname = newvstralloc( holdp[i]->destname,
3384 holdingdisk_get_diskdir(holdp[i]->disk->hdisk), "/",
3385 hd_driver_timestamp, "/",
3386 diskp->host->hostname, ".",
3389 sched(diskp)->holdp[j++] = holdp[i];
3390 holdp[i]->disk->allocated_space += holdp[i]->reserved;
3391 size = (holdp[i]->reserved > size) ? (off_t)0 :
3392 (size - holdp[i]->reserved);
3393 qname = quote_string(diskp->name);
3395 _("assign_holdingdisk: %d assigning holding disk %s to disk %s:%s, reserved %lld, left %lld\n"),
3396 i, holdingdisk_get_diskdir(holdp[i]->disk->hdisk),
3397 diskp->host->hostname, qname,
3398 (long long)holdp[i]->reserved,
3401 holdp[i] = NULL; /* so it doesn't get free()d... */
3403 sched(diskp)->holdp[j] = NULL;
3414 assignedhd_t **holdp;
3415 off_t total = (off_t)0;
3418 char *qname, *hqname, *qdest;
3420 (void)cmd; /* Quiet unused parameter warning */
3422 qname = quote_string(diskp->name);
3423 qdest = quote_string(sched(diskp)->destname);
3424 hold_debug(1, _("adjust_diskspace: %s:%s %s\n"),
3425 diskp->host->hostname, qname, qdest);
3427 holdp = sched(diskp)->holdp;
3429 assert(holdp != NULL);
3431 for( i = 0; holdp[i]; i++ ) { /* for each allocated disk */
3432 diff = holdp[i]->used - holdp[i]->reserved;
3433 total += holdp[i]->used;
3434 holdp[i]->disk->allocated_space += diff;
3435 hqname = quote_string(holdingdisk_name(holdp[i]->disk->hdisk));
3436 hold_debug(1, _("adjust_diskspace: hdisk %s done, reserved %lld used %lld diff %lld alloc %lld dumpers %d\n"),
3437 holdingdisk_name(holdp[i]->disk->hdisk),
3438 (long long)holdp[i]->reserved,
3439 (long long)holdp[i]->used,
3441 (long long)holdp[i]->disk->allocated_space,
3442 holdp[i]->disk->allocated_dumpers );
3443 holdp[i]->reserved += diff;
3447 sched(diskp)->act_size = total;
3449 hold_debug(1, _("adjust_diskspace: after: disk %s:%s used %lld\n"),
3450 diskp->host->hostname, qname,
3451 (long long)sched(diskp)->act_size);
3460 assignedhd_t **holdp;
3463 holdp = sched(diskp)->holdp;
3465 assert(holdp != NULL);
3467 for( i = 0; holdp[i]; i++ ) { /* for each disk */
3468 /* find all files of this dump on that disk, and subtract their
3469 * reserved sizes from the disk's allocated space
3471 holdp[i]->disk->allocated_space -= holdp[i]->used;
3474 holding_file_unlink(holdp[0]->destname); /* no need for the entire list,
3475 * because holding_file_unlink
3476 * will walk through all files
3477 * using cont_filename */
3478 free_assignedhd(sched(diskp)->holdp);
3479 sched(diskp)->holdp = NULL;
3480 sched(diskp)->act_size = (off_t)0;
3483 static assignedhd_t **
3490 char buffer[DISK_BLOCK_BYTES];
3492 assignedhd_t **result;
3495 char dirname[1000], *ch;
3497 char *filename = destname;
3499 memset(buffer, 0, sizeof(buffer));
3500 used = alloc(SIZEOF(off_t) * num_holdalloc);
3501 for(i=0;i<num_holdalloc;i++)
3503 result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3505 while(filename != NULL && filename[0] != '\0') {
3506 strncpy(dirname, filename, 999);
3508 ch = strrchr(dirname,'/');
3511 ch = strrchr(dirname,'/');
3518 g_fprintf(stderr,_("build_diskspace: bogus filename '%s'\n"), filename);
3524 for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3525 if(strcmp(dirname, holdingdisk_get_diskdir(ha->hdisk))==0) {
3530 if(stat(filename, &finfo) == -1) {
3531 g_fprintf(stderr, _("stat %s: %s\n"), filename, strerror(errno));
3532 finfo.st_size = (off_t)0;
3534 used[j] += ((off_t)finfo.st_size+(off_t)1023)/(off_t)1024;
3535 if((fd = open(filename,O_RDONLY)) == -1) {
3536 g_fprintf(stderr,_("build_diskspace: open of %s failed: %s\n"),
3537 filename, strerror(errno));
3542 if ((buflen = full_read(fd, buffer, SIZEOF(buffer))) > 0) {;
3543 parse_file_header(buffer, &file, buflen);
3546 filename = file.cont_filename;
3549 for(j = 0, i=0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3550 if(used[j] != (off_t)0) {
3551 result[i] = alloc(SIZEOF(assignedhd_t));
3552 result[i]->disk = ha;
3553 result[i]->reserved = used[j];
3554 result[i]->used = used[j];
3555 result[i]->destname = stralloc(destname);
3573 g_printf(_("driver: hdisk-state time %s"), time_str);
3575 for(ha = holdalloc, dsk = 0; ha != NULL; ha = ha->next, dsk++) {
3576 diff = ha->disksize - ha->allocated_space;
3577 g_printf(_(" hdisk %d: free %lld dumpers %d"), dsk,
3578 (long long)diff, ha->allocated_dumpers);
3587 time_t save_timestamp = sched(dp)->timestamp;
3588 /* setting timestamp to 0 removes the current level from the
3589 * database, so that we ensure that it will not be bumped to the
3590 * next level on the next run. If we didn't do this, dumpdates or
3591 * gnutar-lists might have been updated already, and a bumped
3592 * incremental might be created. */
3593 sched(dp)->timestamp = 0;
3594 update_info_dumper(dp, (off_t)-1, (off_t)-1, (time_t)-1);
3595 sched(dp)->timestamp = save_timestamp;
3598 /* ------------------- */
3607 for(len = 0, p = q.head; p != NULL; len++, p = p->next)
3608 (void)len; /* Quiet lint */
3613 short_dump_state(void)
3618 wall_time = walltime_str(curclock());
3620 g_printf(_("driver: state time %s "), wall_time);
3621 g_printf(_("free kps: %lu space: %lld taper: "),
3623 (long long)free_space());
3624 if(degraded_mode) g_printf(_("DOWN"));
3628 for(taper = tapetable; taper < tapetable+conf_taper_parallel_write;
3630 if (taper->state & TAPER_STATE_DUMP_TO_TAPE ||
3631 taper->state & TAPER_STATE_FILE_TO_TAPE)
3635 g_printf(_("writing"));
3637 g_printf(_("idle"));
3640 for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
3641 g_printf(_(" idle-dumpers: %d"), nidle);
3642 g_printf(_(" qlen tapeq: %d"), queue_length(tapeq));
3643 g_printf(_(" runq: %d"), queue_length(runq));
3644 g_printf(_(" roomq: %d"), queue_length(roomq));
3645 g_printf(_(" wakeup: %d"), (int)sleep_time);
3646 g_printf(_(" driver-idle: %s\n"), _(idle_strings[idle_reason]));
3647 interface_state(wall_time);
3648 holdingdisk_state(wall_time);
3655 char **why_no_new_tape)
3657 TapeAction result = TAPE_ACTION_NO_ACTION;
3666 off_t dump_to_disk_size;
3667 int dump_to_disk_terminated;
3668 off_t my_flush_threshold_dumped;
3669 off_t my_flush_threshold_scheduled;
3670 off_t my_taperflush;
3671 int nb_taper_active = nb_sent_new_tape;
3672 int nb_taper_flushing = 0;
3675 for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
3676 if (dumper->busy && !sched(dumper->dp)->taper)
3677 dumpers_size += sched(dumper->dp)->est_size;
3679 driver_debug(1, _("dumpers_size: %lld\n"), (long long)dumpers_size);
3682 for(dp = runq.head; dp != NULL; dp = dp->next) {
3683 runq_size += sched(dp)->est_size;
3685 driver_debug(1, _("runq_size: %lld\n"), (long long)runq_size);
3688 for(dp = directq.head; dp != NULL; dp = dp->next) {
3689 directq_size += sched(dp)->est_size;
3691 driver_debug(1, _("directq_size: %lld\n"), (long long)directq_size);
3693 tapeq_size = directq_size;
3694 for(dp = tapeq.head; dp != NULL; dp = dp->next) {
3695 tapeq_size += sched(dp)->act_size;
3698 for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3700 if (taper1->state & TAPER_STATE_FILE_TO_TAPE ||
3701 taper1->state & TAPER_STATE_DUMP_TO_TAPE) {
3702 nb_taper_flushing++;
3706 /* Add what is currently written to tape and in the go. */
3707 for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3709 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3710 tapeq_size -= taper1->left;
3713 if (taper1->dumper) {
3714 tapeq_size += sched(taper1->disk)->est_size - taper1->written;
3716 tapeq_size += sched(taper1->disk)->act_size - taper1->written;
3720 driver_debug(1, _("tapeq_size: %lld\n"), (long long)tapeq_size);
3722 sched_size = runq_size + directq_size + tapeq_size + dumpers_size;
3723 driver_debug(1, _("sched_size: %lld\n"), (long long)sched_size);
3725 dump_to_disk_size = dumpers_size + runq_size + directq_size;
3726 driver_debug(1, _("dump_to_disk_size: %lld\n"), (long long)dump_to_disk_size);
3728 dump_to_disk_terminated = schedule_done && dump_to_disk_size == 0;
3730 for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
3732 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3736 if (nb_taper_active >= 1) {
3737 my_flush_threshold_dumped = flush_threshold_dumped +
3738 (nb_taper_active-nb_taper_active) * tape_length;
3739 my_flush_threshold_scheduled = flush_threshold_scheduled +
3740 (nb_taper_active-nb_taper_active) * tape_length;
3741 my_taperflush = taperflush + (nb_taper_active-nb_taper_active) * tape_length;
3743 my_flush_threshold_dumped = flush_threshold_dumped +
3744 nb_taper_active * tape_length;
3745 my_flush_threshold_scheduled = flush_threshold_scheduled +
3746 nb_taper_active * tape_length;
3747 my_taperflush = taperflush + nb_taper_active * tape_length;
3750 // Changing conditionals can produce a driver hang, take care.
3752 // when to start writting to a new tape
3753 if (taper->state & TAPER_STATE_TAPE_REQUESTED) {
3754 if (current_tape >= conf_runtapes && taper_nb_scan_volume == 0 &&
3755 nb_taper_active == 0) {
3756 *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3757 "does not allow additional tapes"), current_tape, conf_runtapes);
3758 result |= TAPE_ACTION_NO_NEW_TAPE;
3759 } else if (current_tape < conf_runtapes &&
3760 taper_nb_scan_volume == 0 &&
3761 ((my_flush_threshold_dumped < tapeq_size &&
3762 my_flush_threshold_scheduled < sched_size) ||
3763 nb_taper_active == 0) &&
3764 (last_started_taper == NULL ||
3765 last_started_taper == taper)) {
3766 result |= TAPE_ACTION_SCAN;
3768 result |= TAPE_ACTION_MOVE;
3770 } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3771 ((taper->state & TAPER_STATE_DUMP_TO_TAPE) || // for dump to tape
3772 !empty(directq) || // if a dle is waiting for a dump to tape
3773 !empty(roomq) || // holding disk constraint
3774 idle_reason == IDLE_NO_DISKSPACE || // holding disk constraint
3775 (my_flush_threshold_dumped < tapeq_size && // flush-threshold-dumped &&
3776 my_flush_threshold_scheduled < sched_size) || // flush-threshold-scheduled
3777 (my_taperflush < tapeq_size && // taperflush
3778 (force_flush == 1 || // if force_flush
3779 dump_to_disk_terminated)) // or all dump to disk terminated
3781 result |= TAPE_ACTION_NEW_TAPE;
3782 // when to stop using new tape
3783 } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3784 (my_taperflush >= tapeq_size && // taperflush criteria
3785 (force_flush == 1 || // if force_flush
3786 dump_to_disk_terminated)) // or all dump to disk
3788 if (nb_taper_active <= 0) {
3789 if (current_tape >= conf_runtapes) {
3790 *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3791 "does not allow additional tapes"), current_tape, conf_runtapes);
3793 *why_no_new_tape = _("taperflush criteria not met");
3795 result |= TAPE_ACTION_NO_NEW_TAPE;
3799 // when to start a flush
3800 if (taper->state & TAPER_STATE_IDLE) {
3801 if (!degraded_mode && (!empty(tapeq) || !empty(directq)) &&
3802 (taper->state & TAPER_STATE_TAPE_STARTED || // tape already started
3803 !empty(roomq) || // holding disk constraint
3804 idle_reason == IDLE_NO_DISKSPACE || // holding disk constraint
3805 (my_flush_threshold_dumped < tapeq_size && // flush-threshold-dumped &&
3806 my_flush_threshold_scheduled < sched_size) || // flush-threshold-scheduled
3807 (force_flush == 1 && my_taperflush < tapeq_size))) { // taperflush if force_flush
3809 if (nb_taper_flushing == 0) {
3810 result |= TAPE_ACTION_START_A_FLUSH;
3812 result |= TAPE_ACTION_START_A_FLUSH_FIT;
3820 all_taper_idle(void)
3824 for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
3826 if (taper->state & TAPER_STATE_DUMP_TO_TAPE ||
3827 taper->state & TAPER_STATE_FILE_TO_TAPE)
3842 g_printf("================\n");
3843 g_printf(_("driver state at time %s: %s\n"), walltime_str(curclock()), str);
3844 g_printf(_("free kps: %lu, space: %lld\n"),
3846 (long long)free_space());
3847 if(degraded_mode) g_printf(_("taper: DOWN\n"));
3848 else if(taper->status == TAPER_IDLE) g_printf(_("taper: idle\n"));
3849 else g_printf(_("taper: writing %s:%s.%d est size %lld\n"),
3850 taper->disk->host->hostname, taper->disk->name,
3851 sched(taper->disk)->level,
3852 (long long)sched(taper->disk)->est_size);
3853 for(i = 0; i < inparallel; i++) {
3854 dp = dmptable[i].dp;
3855 if(!dmptable[i].busy)
3856 g_printf(_("%s: idle\n"), dmptable[i].name);
3858 qname = quote_string(dp->name);
3859 g_printf(_("%s: dumping %s:%s.%d est kps %d size %lld time %lu\n"),
3860 dmptable[i].name, dp->host->hostname, qname, sched(dp)->level,
3861 sched(dp)->est_kps, (long long)sched(dp)->est_size, sched(dp)->est_time);
3864 dump_queue("TAPE", tapeq, 5, stdout);
3865 dump_queue("ROOM", roomq, 5, stdout);
3866 dump_queue("RUN ", runq, 5, stdout);
3867 g_printf("================\n");