git.gag.com Git - debian/amanda/blob - server-src/driver.c

   1 /*
   2  * Amanda, The Advanced Maryland Automatic Network Disk Archiver
   3  * Copyright (c) 1991-1998 University of Maryland at College Park
   4  * Copyright (c) 2007-2012 Zmanda, Inc.  All Rights Reserved.
   5  * All Rights Reserved.
   6  *
   7  * Permission to use, copy, modify, distribute, and sell this software and its
   8  * documentation for any purpose is hereby granted without fee, provided that
   9  * the above copyright notice appear in all copies and that both that
  10  * copyright notice and this permission notice appear in supporting
  11  * documentation, and that the name of U.M. not be used in advertising or
  12  * publicity pertaining to distribution of the software without specific,
  13  * written prior permission.  U.M. makes no representations about the
  14  * suitability of this software for any purpose.  It is provided "as is"
  15  * without express or implied warranty.
  16  *
  17  * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
  18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
  19  * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  20  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  21  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  22  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  23  *
  24  * Authors: the Amanda Development Team.  Its members are listed in a
  25  * file named AUTHORS, in the root directory of this distribution.
  26  */
  27 /*
  28  * $Id: driver.c 6512 2007-05-24 17:00:24Z ian $
  29  *
  30  * controlling process for the Amanda backup system
  31  */
  32
  33 /*
  34  * XXX possibly modify tape queue to be cognizant of how much room is left on
  35  *     tape.  Probably not effective though, should do this in planner.
  36  */
  37
  38 #include "amanda.h"
  39 #include "find.h"
  40 #include "clock.h"
  41 #include "conffile.h"
  42 #include "diskfile.h"
  43 #include "event.h"
  44 #include "holding.h"
  45 #include "infofile.h"
  46 #include "logfile.h"
  47 #include "fsusage.h"
  48 #include "driverio.h"
  49 #include "server_util.h"
  50 #include "timestamp.h"
  51
  52 #define driver_debug(i, ...) do {       \
  53         if ((i) <= debug_driver) {      \
  54             dbprintf(__VA_ARGS__);      \
  55         }                               \
  56 } while (0)
  57
  58 #define hold_debug(i, ...) do {         \
  59         if ((i) <= debug_holding) {     \
  60             dbprintf(__VA_ARGS__);      \
  61         }                               \
  62 } while (0)
  63
  64 static disklist_t waitq;        // dle waiting estimate result
  65 static disklist_t runq;         // dle waiting to be dumped to holding disk
  66 static disklist_t directq;      // dle waiting to be dumped directly to tape
  67 static disklist_t tapeq;        // dle on holding disk waiting to be written
  68                                 //   to tape
  69 static disklist_t roomq;        // dle waiting for more space on holding disk
  70 static int pending_aborts;
  71 static int degraded_mode;
  72 static off_t reserved_space;
  73 static off_t total_disksize;
  74 static char *dumper_program;
  75 static char *chunker_program;
  76 static int  inparallel;
  77 static int nodump = 0;
  78 static off_t tape_length = (off_t)0;
  79 static int current_tape = 0;
  80 static int conf_max_dle_by_volume;
  81 static int conf_taperalgo;
  82 static int conf_taper_parallel_write;
  83 static int conf_runtapes;
  84 static time_t sleep_time;
  85 static int idle_reason;
  86 static char *driver_timestamp;
  87 static char *hd_driver_timestamp;
  88 static am_host_t *flushhost = NULL;
  89 static int need_degraded=0;
  90 static holdalloc_t *holdalloc;
  91 static int num_holdalloc;
  92 static event_handle_t *dumpers_ev_time = NULL;
  93 static event_handle_t *flush_ev_read = NULL;
  94 static event_handle_t *schedule_ev_read = NULL;
  95 static int   conf_flush_threshold_dumped;
  96 static int   conf_flush_threshold_scheduled;
  97 static int   conf_taperflush;
  98 static off_t flush_threshold_dumped;
  99 static off_t flush_threshold_scheduled;
 100 static off_t taperflush;
 101 static int   schedule_done;                     // 1 if we don't wait for a
 102                                                 //   schedule from the planner
 103 static int   force_flush;                       // All dump are terminated, we
 104                                                 // must now respect taper_flush
 105 static int taper_nb_scan_volume = 0;
 106 static int nb_sent_new_tape = 0;
 107 static int taper_started = 0;
 108 static taper_t *last_started_taper;
 109
 110 static int wait_children(int count);
 111 static void wait_for_children(void);
 112 static void allocate_bandwidth(netif_t *ip, unsigned long kps);
 113 static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp);
 114 static void adjust_diskspace(disk_t *diskp, cmd_t cmd);
 115 static void delete_diskspace(disk_t *diskp);
 116 static assignedhd_t **build_diskspace(char *destname);
 117 static int client_constrained(disk_t *dp);
 118 static void deallocate_bandwidth(netif_t *ip, unsigned long kps);
 119 static void dump_schedule(disklist_t *qp, char *str);
 120 static assignedhd_t **find_diskspace(off_t size, int *cur_idle,
 121                                         assignedhd_t *preferred);
 122 static unsigned long free_kps(netif_t *ip);
 123 static off_t free_space(void);
 124 static void dumper_chunker_result(disk_t *dp);
 125 static void dumper_taper_result(disk_t *dp);
 126 static void file_taper_result(disk_t *dp);
 127 static void handle_dumper_result(void *);
 128 static void handle_chunker_result(void *);
 129 static void handle_dumpers_time(void *);
 130 static void handle_taper_result(void *);
 131
 132 static void holdingdisk_state(char *time_str);
 133 static taper_t *idle_taper(void);
 134 static taper_t *taper_from_name(char *name);
 135 static void interface_state(char *time_str);
 136 static int queue_length(disklist_t q);
 137 static void read_flush(void *cookie);
 138 static void read_schedule(void *cookie);
 139 static void short_dump_state(void);
 140 static void startaflush(void);
 141 static void start_degraded_mode(disklist_t *queuep);
 142 static void start_some_dumps(disklist_t *rq);
 143 static void continue_port_dumps(void);
 144 static void update_failed_dump(disk_t *);
 145 static int no_taper_flushing(void);
 146 static int active_dumper(void);
 147
 148 typedef enum {
 149     TAPE_ACTION_NO_ACTION         = 0,
 150     TAPE_ACTION_SCAN              = (1 << 0),
 151     TAPE_ACTION_NEW_TAPE          = (1 << 1),
 152     TAPE_ACTION_NO_NEW_TAPE       = (1 << 2),
 153     TAPE_ACTION_START_A_FLUSH     = (1 << 3),
 154     TAPE_ACTION_START_A_FLUSH_FIT = (1 << 4),
 155     TAPE_ACTION_MOVE              = (1 << 5)
 156 } TapeAction;
 157
 158 static TapeAction tape_action(taper_t *taper, char **why_no_new_tape);
 159
 160 static const char *idle_strings[] = {
 161 #define NOT_IDLE                0
 162     T_("not-idle"),
 163 #define IDLE_NO_DUMPERS         1
 164     T_("no-dumpers"),
 165 #define IDLE_START_WAIT         2
 166     T_("start-wait"),
 167 #define IDLE_NO_HOLD            3
 168     T_("no-hold"),
 169 #define IDLE_CLIENT_CONSTRAINED 4
 170     T_("client-constrained"),
 171 #define IDLE_NO_BANDWIDTH       5
 172     T_("no-bandwidth"),
 173 #define IDLE_NO_DISKSPACE       6
 174     T_("no-diskspace")
 175 };
 176
 177 int
 178 main(
 179     int         argc,
 180     char **     argv)
 181 {
 182     disklist_t origq;
 183     disk_t *diskp;
 184     int dsk;
 185     dumper_t *dumper;
 186     char *newdir = NULL;
 187     struct fs_usage fsusage;
 188     holdingdisk_t *hdp;
 189     identlist_t    il;
 190     unsigned long reserve = 100;
 191     char *conf_diskfile;
 192     char *taper_program;
 193     char *conf_tapetype;
 194     tapetype_t *tape;
 195     char *line;
 196     char hostname[1025];
 197     intmax_t kb_avail;
 198     config_overrides_t *cfg_ovr = NULL;
 199     char *cfg_opt = NULL;
 200     holdalloc_t *ha, *ha_last;
 201     find_result_t *holding_files;
 202     disklist_t holding_disklist = { NULL, NULL };
 203     int no_taper = FALSE;
 204     int from_client = FALSE;
 205
 206     if (argc > 1 && argv && argv[1] && g_str_equal(argv[1], "--version")) {
 207         printf("driver-%s\n", VERSION);
 208         return (0);
 209     }
 210
 211     /*
 212      * Configure program for internationalization:
 213      *   1) Only set the message locale for now.
 214      *   2) Set textdomain for all amanda related programs to "amanda"
 215      *      We don't want to be forced to support dozens of message catalogs.
 216      */
 217     setlocale(LC_MESSAGES, "C");
 218     textdomain("amanda");
 219
 220     safe_fd(-1, 0);
 221
 222     setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0);
 223     setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0);
 224
 225     set_pname("driver");
 226
 227     dbopen(DBG_SUBDIR_SERVER);
 228
 229     atexit(wait_for_children);
 230
 231     /* Don't die when child closes pipe */
 232     signal(SIGPIPE, SIG_IGN);
 233
 234     add_amanda_log_handler(amanda_log_stderr);
 235     add_amanda_log_handler(amanda_log_trace_log);
 236
 237     startclock();
 238
 239     cfg_ovr = extract_commandline_config_overrides(&argc, &argv);
 240
 241     if (argc > 1)
 242         cfg_opt = argv[1];
 243     set_config_overrides(cfg_ovr);
 244     config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt);
 245
 246     conf_diskfile = config_dir_relative(getconf_str(CNF_DISKFILE));
 247     read_diskfile(conf_diskfile, &origq);
 248     disable_skip_disk(&origq);
 249     amfree(conf_diskfile);
 250
 251     if (config_errors(NULL) >= CFGERR_WARNINGS) {
 252         config_print_errors();
 253         if (config_errors(NULL) >= CFGERR_ERRORS) {
 254             g_critical(_("errors processing config file"));
 255         }
 256     }
 257
 258     log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid());
 259     g_printf(_("%s: pid %ld executable %s version %s\n"),
 260            get_pname(), (long) getpid(), argv[0], VERSION);
 261
 262     if(argc > 2) {
 263         if(strcmp(argv[2], "nodump") == 0) {
 264             nodump = 1;
 265             argv++;
 266             argc--;
 267         }
 268     }
 269
 270     if (argc > 2) {
 271         if (strcmp(argv[2], "--no-taper") == 0) {
 272             no_taper = TRUE;
 273             argv++;
 274             argc--;
 275         }
 276     }
 277
 278     if (argc > 2) {
 279         if (strcmp(argv[2], "--from-client") == 0) {
 280             from_client = TRUE;
 281             from_client = from_client;
 282             argv++;
 283             argc--;
 284         }
 285     }
 286
 287     safe_cd(); /* do this *after* config_init */
 288
 289     check_running_as(RUNNING_AS_DUMPUSER);
 290
 291     dbrename(get_config_name(), DBG_SUBDIR_SERVER);
 292
 293     /* load DLEs from the holding disk, in case there's anything to flush there */
 294     search_holding_disk(&holding_files, &holding_disklist);
 295     /* note that the dumps are added to the global disklist, so we need not consult
 296      * holding_files or holding_disklist after this */
 297
 298     amfree(driver_timestamp);
 299     /* read timestamp from stdin */
 300     while ((line = agets(stdin)) != NULL) {
 301         if (line[0] != '\0')
 302             break;
 303         amfree(line);
 304     }
 305     if ( line == NULL ) {
 306       error(_("Did not get DATE line from planner"));
 307       /*NOTREACHED*/
 308     }
 309     driver_timestamp = alloc(15);
 310     strncpy(driver_timestamp, &line[5], 14);
 311     driver_timestamp[14] = '\0';
 312     amfree(line);
 313     log_add(L_START,_("date %s"), driver_timestamp);
 314
 315     gethostname(hostname, SIZEOF(hostname));
 316     log_add(L_STATS,_("hostname %s"), hostname);
 317
 318     /* check that we don't do many dump in a day and usetimestamps is off */
 319     if(strlen(driver_timestamp) == 8) {
 320         if (!nodump) {
 321             char *conf_logdir = getconf_str(CNF_LOGDIR);
 322             char *logfile    = vstralloc(conf_logdir, "/log.",
 323                                          driver_timestamp, ".0", NULL);
 324             char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.",
 325                                          driver_timestamp, ".0", NULL);
 326             if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) {
 327                 log_add(L_WARNING, _("WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day."));
 328             }
 329             amfree(oldlogfile);
 330             amfree(logfile);
 331         }
 332         hd_driver_timestamp = get_timestamp_from_time(0);
 333     }
 334     else {
 335         hd_driver_timestamp = stralloc(driver_timestamp);
 336     }
 337
 338     taper_program = vstralloc(amlibexecdir, "/", "taper", NULL);
 339     dumper_program = vstralloc(amlibexecdir, "/", "dumper", NULL);
 340     chunker_program = vstralloc(amlibexecdir, "/", "chunker", NULL);
 341
 342     conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO);
 343     conf_taper_parallel_write = getconf_int(CNF_TAPER_PARALLEL_WRITE);
 344     conf_tapetype = getconf_str(CNF_TAPETYPE);
 345     conf_runtapes = getconf_int(CNF_RUNTAPES);
 346     conf_max_dle_by_volume = getconf_int(CNF_MAX_DLE_BY_VOLUME);
 347     if (conf_taper_parallel_write > conf_runtapes) {
 348         conf_taper_parallel_write = conf_runtapes;
 349     }
 350     tape = lookup_tapetype(conf_tapetype);
 351     tape_length = tapetype_get_length(tape);
 352     g_printf("driver: tape size %lld\n", (long long)tape_length);
 353     conf_flush_threshold_dumped = getconf_int(CNF_FLUSH_THRESHOLD_DUMPED);
 354     conf_flush_threshold_scheduled = getconf_int(CNF_FLUSH_THRESHOLD_SCHEDULED);
 355     conf_taperflush = getconf_int(CNF_TAPERFLUSH);
 356     flush_threshold_dumped = (conf_flush_threshold_dumped * tape_length) / 100;
 357     flush_threshold_scheduled = (conf_flush_threshold_scheduled * tape_length) / 100;
 358     taperflush = (conf_taperflush *tape_length) / 100;
 359
 360     driver_debug(1, _("flush-threshold-dumped: %lld\n"), (long long)flush_threshold_dumped);
 361     driver_debug(1, _("flush-threshold-scheduled: %lld\n"), (long long)flush_threshold_scheduled);
 362     driver_debug(1, _("taperflush: %lld\n"), (long long)taperflush);
 363
 364     /* set up any configuration-dependent variables */
 365
 366     inparallel  = getconf_int(CNF_INPARALLEL);
 367
 368     reserve = (unsigned long)getconf_int(CNF_RESERVE);
 369
 370     total_disksize = (off_t)0;
 371     ha_last = NULL;
 372     num_holdalloc = 0;
 373     for (il = getconf_identlist(CNF_HOLDINGDISK), dsk = 0;
 374          il != NULL;
 375          il = il->next, dsk++) {
 376         hdp = lookup_holdingdisk(il->data);
 377         ha = alloc(SIZEOF(holdalloc_t));
 378         num_holdalloc++;
 379
 380         /* link the list in the same order as getconf_holdingdisks's results */
 381         ha->next = NULL;
 382         if (ha_last == NULL)
 383             holdalloc = ha;
 384         else
 385             ha_last->next = ha;
 386         ha_last = ha;
 387
 388         ha->hdisk = hdp;
 389         ha->allocated_dumpers = 0;
 390         ha->allocated_space = (off_t)0;
 391         ha->disksize = holdingdisk_get_disksize(hdp);
 392
 393         /* get disk size */
 394         if(get_fs_usage(holdingdisk_get_diskdir(hdp), NULL, &fsusage) == -1
 395            || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) {
 396             log_add(L_WARNING, _("WARNING: ignoring holding disk %s: %s\n"),
 397                     holdingdisk_get_diskdir(hdp), strerror(errno));
 398             ha->disksize = 0L;
 399             continue;
 400         }
 401
 402         /* do the division first to avoid potential integer overflow */
 403         if (fsusage.fsu_bavail_top_bit_set)
 404             kb_avail = 0;
 405         else
 406             kb_avail = fsusage.fsu_bavail / 1024 * fsusage.fsu_blocksize;
 407
 408         if(ha->disksize > (off_t)0) {
 409             if(ha->disksize > kb_avail) {
 410                 log_add(L_WARNING,
 411                         _("WARNING: %s: %lld KB requested, "
 412                         "but only %lld KB available."),
 413                         holdingdisk_get_diskdir(hdp),
 414                         (long long)ha->disksize,
 415                         (long long)kb_avail);
 416                         ha->disksize = kb_avail;
 417             }
 418         }
 419         /* ha->disksize is negative; use all but that amount */
 420         else if(kb_avail < -ha->disksize) {
 421             log_add(L_WARNING,
 422                     _("WARNING: %s: not %lld KB free."),
 423                     holdingdisk_get_diskdir(hdp),
 424                     (long long)-ha->disksize);
 425             ha->disksize = (off_t)0;
 426             continue;
 427         }
 428         else
 429             ha->disksize += kb_avail;
 430
 431         g_printf(_("driver: adding holding disk %d dir %s size %lld chunksize %lld\n"),
 432                dsk, holdingdisk_get_diskdir(hdp),
 433                (long long)ha->disksize,
 434                (long long)(holdingdisk_get_chunksize(hdp)));
 435
 436         newdir = newvstralloc(newdir,
 437                               holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp,
 438                               NULL);
 439         if(!mkholdingdir(newdir)) {
 440             ha->disksize = (off_t)0;
 441         }
 442         total_disksize += ha->disksize;
 443     }
 444
 445     reserved_space = total_disksize * (off_t)(reserve / 100);
 446
 447     g_printf(_("reserving %lld out of %lld for degraded-mode dumps\n"),
 448            (long long)reserved_space, (long long)free_space());
 449
 450     amfree(newdir);
 451
 452     if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS;
 453
 454     /* taper takes a while to get going, so start it up right away */
 455
 456     init_driverio();
 457     startup_tape_process(taper_program, conf_taper_parallel_write, no_taper);
 458
 459     /* fire up the dumpers now while we are waiting */
 460     if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp);
 461
 462     /*
 463      * Read schedule from stdin.  Usually, this is a pipe from planner,
 464      * so the effect is that we wait here for the planner to
 465      * finish, but meanwhile the taper is rewinding the tape, reading
 466      * the label, checking it, writing a new label and all that jazz
 467      * in parallel with the planner.
 468      */
 469
 470     runq.head = NULL;
 471     runq.tail = NULL;
 472     directq.head = NULL;
 473     directq.tail = NULL;
 474     waitq = origq;
 475     tapeq.head = NULL;
 476     tapeq.tail = NULL;
 477     roomq.head = NULL;
 478     roomq.tail = NULL;
 479     taper_nb_wait_reply = 0;
 480
 481     need_degraded = 0;
 482     if (no_taper || conf_runtapes <= 0) {
 483         taper_started = 1; /* we'll pretend the taper started and failed immediately */
 484         need_degraded = 1;
 485     } else {
 486         tapetable[0].state = TAPER_STATE_INIT;
 487         taper_nb_wait_reply++;
 488         taper_nb_scan_volume++;
 489         taper_ev_read = event_register(taper_fd, EV_READFD,
 490                                        handle_taper_result, NULL);
 491         taper_cmd(START_TAPER, NULL, tapetable[0].name, 0, driver_timestamp);
 492     }
 493
 494     flush_ev_read = event_register((event_id_t)0, EV_READFD, read_flush, NULL);
 495
 496     log_add(L_STATS, _("startup time %s"), walltime_str(curclock()));
 497
 498     g_printf(_("driver: start time %s inparallel %d bandwidth %lu diskspace %lld "), walltime_str(curclock()), inparallel,
 499            free_kps(NULL), (long long)free_space());
 500     g_printf(_(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n"),
 501            "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo),
 502            getconf_str(CNF_DUMPORDER));
 503     fflush(stdout);
 504
 505     schedule_done = nodump;
 506     force_flush = 0;
 507
 508     short_dump_state();
 509     event_loop(0);
 510
 511     force_flush = 1;
 512
 513     /* mv runq to directq */
 514     while (!empty(runq)) {
 515         diskp = dequeue_disk(&runq);
 516         headqueue_disk(&directq, diskp);
 517     }
 518
 519     run_server_global_scripts(EXECUTE_ON_POST_BACKUP, get_config_name());
 520
 521     /* log error for any remaining dumps */
 522     while(!empty(directq)) {
 523         diskp = dequeue_disk(&directq);
 524
 525         if (diskp->to_holdingdisk == HOLD_REQUIRED) {
 526             char *qname = quote_string(diskp->name);
 527             log_add(L_FAIL, "%s %s %s %d [%s]",
 528                 diskp->host->hostname, qname, sched(diskp)->datestamp,
 529                 sched(diskp)->level,
 530                 _("can't dump required holdingdisk"));
 531             amfree(qname);
 532         }
 533         else if (!degraded_mode) {
 534             char *qname = quote_string(diskp->name);
 535             log_add(L_FAIL, "%s %s %s %d [%s]",
 536                 diskp->host->hostname, qname, sched(diskp)->datestamp,
 537                 sched(diskp)->level,
 538                 _("can't dump in non degraded mode"));
 539             amfree(qname);
 540         }
 541         else {
 542             char *qname = quote_string(diskp->name);
 543             log_add(L_FAIL, "%s %s %s %d [%s]",
 544                 diskp->host->hostname, qname, sched(diskp)->datestamp,
 545                 sched(diskp)->level,
 546                 num_holdalloc == 0 ?
 547                     _("can't do degraded dump without holding disk") :
 548                 diskp->to_holdingdisk != HOLD_NEVER ?
 549                     _("out of holding space in degraded mode") :
 550                     _("can't dump 'holdingdisk never' dle in degraded mode"));
 551             amfree(qname);
 552         }
 553     }
 554
 555     short_dump_state();                         /* for amstatus */
 556
 557     g_printf(_("driver: QUITTING time %s telling children to quit\n"),
 558            walltime_str(curclock()));
 559     fflush(stdout);
 560
 561     if(!nodump) {
 562         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 563             if(dumper->fd >= 0)
 564                 dumper_cmd(dumper, QUIT, NULL, NULL);
 565         }
 566     }
 567
 568     if(taper_fd >= 0) {
 569         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 570     }
 571
 572     /* wait for all to die */
 573     wait_children(600);
 574
 575     /* cleanup */
 576     holding_cleanup(NULL, NULL);
 577
 578     amfree(newdir);
 579
 580     check_unfree_serial();
 581     g_printf(_("driver: FINISHED time %s\n"), walltime_str(curclock()));
 582     fflush(stdout);
 583     log_add(L_FINISH,_("date %s time %s"), driver_timestamp, walltime_str(curclock()));
 584     log_add(L_INFO, "pid-done %ld", (long)getpid());
 585     amfree(driver_timestamp);
 586
 587     amfree(dumper_program);
 588     amfree(taper_program);
 589
 590     dbclose();
 591
 592     return 0;
 593 }
 594
 595 /* sleep up to count seconds, and wait for terminating child process */
 596 /* if sleep is negative, this function will not timeout              */
 597 /* exit once all child process are finished or the timout expired    */
 598 /* return 0 if no more children to wait                              */
 599 /* return 1 if some children are still alive                         */
 600 static int
 601 wait_children(int count)
 602 {
 603     pid_t     pid;
 604     amwait_t  retstat;
 605     char     *who;
 606     char     *what;
 607     int       code=0;
 608     dumper_t *dumper;
 609     int       wait_errno;
 610
 611     do {
 612         do {
 613             pid = waitpid((pid_t)-1, &retstat, WNOHANG);
 614             wait_errno = errno;
 615             if (pid > 0) {
 616                 what = NULL;
 617                 if (! WIFEXITED(retstat)) {
 618                     what = _("signal");
 619                     code = WTERMSIG(retstat);
 620                 } else if (WEXITSTATUS(retstat) != 0) {
 621                     what = _("code");
 622                     code = WEXITSTATUS(retstat);
 623                 }
 624                 who = NULL;
 625                 for (dumper = dmptable; dumper < dmptable + inparallel;
 626                      dumper++) {
 627                     if (pid == dumper->pid) {
 628                         who = stralloc(dumper->name);
 629                         dumper->pid = -1;
 630                         break;
 631                     }
 632                     if (dumper->chunker && pid == dumper->chunker->pid) {
 633                         who = stralloc(dumper->chunker->name);
 634                         dumper->chunker->pid = -1;
 635                         break;
 636                     }
 637                 }
 638                 if (who == NULL && pid == taper_pid) {
 639                     who = stralloc("taper");
 640                     taper_pid = -1;
 641                 }
 642                 if(what != NULL && who == NULL) {
 643                     who = stralloc("unknown");
 644                 }
 645                 if(who && what) {
 646                     log_add(L_WARNING, _("%s pid %u exited with %s %d\n"), who,
 647                             (unsigned)pid, what, code);
 648                     g_printf(_("driver: %s pid %u exited with %s %d\n"), who,
 649                            (unsigned)pid, what, code);
 650                 }
 651                 amfree(who);
 652             }
 653         } while (pid > 0 || wait_errno == EINTR);
 654         if (errno != ECHILD)
 655             sleep(1);
 656         if (count > 0)
 657             count--;
 658     } while ((errno != ECHILD) && (count != 0));
 659     return (errno != ECHILD);
 660 }
 661
 662 static void
 663 kill_children(int signal)
 664 {
 665     dumper_t *dumper;
 666
 667     if(!nodump) {
 668         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 669             if (!dumper->down && dumper->pid > 1) {
 670                 g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
 671                        dumper->name, (unsigned)dumper->pid);
 672                 if (kill(dumper->pid, signal) == -1 && errno == ESRCH) {
 673                     if (dumper->chunker)
 674                         dumper->chunker->pid = 0;
 675                 }
 676                 if (dumper->chunker && dumper->chunker->pid > 1) {
 677                     g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
 678                            dumper->chunker->name,
 679                            (unsigned)dumper->chunker->pid);
 680                     if (kill(dumper->chunker->pid, signal) == -1 &&
 681                         errno == ESRCH)
 682                         dumper->chunker->pid = 0;
 683                 }
 684             }
 685         }
 686     }
 687
 688     if(taper_pid > 1) {
 689         g_printf(_("driver: sending signal %d to %s pid %u\n"), signal,
 690                "taper", (unsigned)taper_pid);
 691         if (kill(taper_pid, signal) == -1 && errno == ESRCH)
 692             taper_pid = 0;
 693     }
 694 }
 695
 696 static void
 697 wait_for_children(void)
 698 {
 699     dumper_t *dumper;
 700
 701     if(!nodump) {
 702         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 703             if (dumper->pid > 1 && dumper->fd >= 0) {
 704                 dumper_cmd(dumper, QUIT, NULL, NULL);
 705                 if (dumper->chunker && dumper->chunker->pid > 1 &&
 706                     dumper->chunker->fd >= 0)
 707                     chunker_cmd(dumper->chunker, QUIT, NULL, NULL);
 708             }
 709         }
 710     }
 711
 712     if(taper_pid > 1 && taper_fd > 0) {
 713         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 714     }
 715
 716     if(wait_children(60) == 0)
 717         return;
 718
 719     kill_children(SIGHUP);
 720     if(wait_children(60) == 0)
 721         return;
 722
 723     kill_children(SIGKILL);
 724     if(wait_children(-1) == 0)
 725         return;
 726
 727 }
 728
 729 static void startaflush_tape(taper_t *taper, gboolean *state_changed);
 730
 731 static void
 732 startaflush(void)
 733 {
 734     taper_t *taper;
 735     gboolean state_changed = FALSE;
 736
 737     for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
 738         taper++) {
 739         if (!(taper->state & TAPER_STATE_DONE) &&
 740             taper->state & TAPER_STATE_WAIT_FOR_TAPE) {
 741             startaflush_tape(taper, &state_changed);
 742         }
 743     }
 744     for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
 745         taper++) {
 746         if (!(taper->state & TAPER_STATE_DONE) &&
 747             taper->state & TAPER_STATE_TAPE_REQUESTED) {
 748             startaflush_tape(taper, &state_changed);
 749         }
 750     }
 751     for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
 752         taper++) {
 753         if (!(taper->state & TAPER_STATE_DONE) &&
 754             taper->state & TAPER_STATE_INIT) {
 755             startaflush_tape(taper, &state_changed);
 756         }
 757     }
 758     for(taper = tapetable; taper <= tapetable+conf_taper_parallel_write;
 759         taper++) {
 760         if (!(taper->state & TAPER_STATE_DONE) &&
 761             taper->state & TAPER_STATE_IDLE) {
 762             startaflush_tape(taper, &state_changed);
 763         }
 764     }
 765     if (state_changed) {
 766         short_dump_state();
 767     }
 768 }
 769
 770 static void
 771 startaflush_tape(
 772     taper_t  *taper,
 773     gboolean *state_changed)
 774 {
 775     disk_t *dp = NULL;
 776     disk_t *fit = NULL;
 777     char *datestamp;
 778     off_t extra_tapes_size = 0;
 779     off_t taper_left;
 780     char *qname;
 781     TapeAction result_tape_action;
 782     char *why_no_new_tape = NULL;
 783     taper_t *taper1;
 784
 785     result_tape_action = tape_action(taper, &why_no_new_tape);
 786
 787     if (result_tape_action & TAPE_ACTION_SCAN) {
 788         taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
 789         taper->state |= TAPER_STATE_WAIT_FOR_TAPE;
 790         taper_nb_scan_volume++;
 791         taper_cmd(START_SCAN, taper->disk, NULL, 0, NULL);
 792     } else if (result_tape_action & TAPE_ACTION_NEW_TAPE) {
 793         taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
 794         taper->state |= TAPER_STATE_WAIT_NEW_TAPE;
 795         nb_sent_new_tape++;
 796         taper_cmd(NEW_TAPE, taper->disk, NULL, 0, NULL);
 797     } else if (result_tape_action & TAPE_ACTION_NO_NEW_TAPE) {
 798         taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
 799         taper_cmd(NO_NEW_TAPE, taper->disk, why_no_new_tape, 0, NULL);
 800         taper->state |= TAPER_STATE_DONE;
 801         start_degraded_mode(&runq);
 802         *state_changed = TRUE;
 803     } else if (result_tape_action & TAPE_ACTION_MOVE) {
 804         taper_t *taper1 = idle_taper();
 805         if (taper1) {
 806             taper->state &= ~TAPER_STATE_TAPE_REQUESTED;
 807             taper->state &= ~TAPER_STATE_WAIT_FOR_TAPE;
 808             taper_cmd(TAKE_SCRIBE_FROM, taper->disk, taper1->name, 0 , NULL);
 809             taper1->state = TAPER_STATE_DEFAULT;
 810             taper->state |= TAPER_STATE_TAPE_STARTED;
 811             taper->left = taper1->left;
 812             taper->nb_dle++;
 813             if (last_started_taper == taper1) {
 814                 last_started_taper = taper;
 815             }
 816             *state_changed = TRUE;
 817         }
 818     }
 819
 820     if (!degraded_mode &&
 821         taper->state & TAPER_STATE_IDLE &&
 822         !empty(tapeq) &&
 823         (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
 824          result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT)) {
 825
 826         int taperalgo = conf_taperalgo;
 827         if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
 828             if (taperalgo == ALGO_FIRST)
 829                 taperalgo = ALGO_FIRSTFIT;
 830             else if (taperalgo == ALGO_LARGEST)
 831                 taperalgo = ALGO_LARGESTFIT;
 832             else if (taperalgo == ALGO_SMALLEST)
 833                 taperalgo = ALGO_SMALLESTFIT;
 834             else if (taperalgo == ALGO_LAST)
 835                 taperalgo = ALGO_LASTFIT;
 836         }
 837
 838         extra_tapes_size = tape_length * (off_t)(conf_runtapes - current_tape);
 839         for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
 840                                  taper1++) {
 841             if (taper1->state & TAPER_STATE_TAPE_STARTED) {
 842                 extra_tapes_size += taper1->left;
 843             }
 844             dp = taper1->disk;
 845             if (dp) {
 846                 extra_tapes_size -= (sched(dp)->act_size - taper1->written);
 847             }
 848         }
 849
 850         if (taper->state & TAPER_STATE_TAPE_STARTED) {
 851             taper_left = taper->left;
 852         } else {
 853             taper_left = tape_length;
 854         }
 855         dp = NULL;
 856         datestamp = sched(tapeq.head)->datestamp;
 857         switch(taperalgo) {
 858         case ALGO_FIRST:
 859                 dp = dequeue_disk(&tapeq);
 860                 break;
 861         case ALGO_FIRSTFIT:
 862                 fit = tapeq.head;
 863                 while (fit != NULL) {
 864                     if (sched(fit)->act_size <=
 865                              (fit->splitsize ? extra_tapes_size : taper_left) &&
 866                              strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 867                         dp = fit;
 868                         fit = NULL;
 869                     }
 870                     else {
 871                         fit = fit->next;
 872                     }
 873                 }
 874                 if(dp) remove_disk(&tapeq, dp);
 875                 break;
 876         case ALGO_LARGEST:
 877                 fit = dp = tapeq.head;
 878                 while (fit != NULL) {
 879                     if(sched(fit)->act_size > sched(dp)->act_size &&
 880                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 881                         dp = fit;
 882                     }
 883                     fit = fit->next;
 884                 }
 885                 if(dp) remove_disk(&tapeq, dp);
 886                 break;
 887         case ALGO_LARGESTFIT:
 888                 fit = tapeq.head;
 889                 while (fit != NULL) {
 890                     if(sched(fit)->act_size <=
 891                        (fit->splitsize ? extra_tapes_size : taper_left) &&
 892                        (!dp || sched(fit)->act_size > sched(dp)->act_size) &&
 893                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 894                         dp = fit;
 895                     }
 896                     fit = fit->next;
 897                 }
 898                 if(dp) remove_disk(&tapeq, dp);
 899                 break;
 900         case ALGO_SMALLEST:
 901                 fit = dp = tapeq.head;
 902                 while (fit != NULL) {
 903                     if (sched(fit)->act_size < sched(dp)->act_size &&
 904                         strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 905                         dp = fit;
 906                     }
 907                     fit = fit->next;
 908                 }
 909                 if(dp) remove_disk(&tapeq, dp);
 910                 break;
 911         case ALGO_SMALLESTFIT:
 912                 fit = dp = tapeq.head;
 913                 while (fit != NULL) {
 914                     if (sched(fit)->act_size <=
 915                         (fit->splitsize ? extra_tapes_size : taper_left) &&
 916                         (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
 917                         strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 918                         dp = fit;
 919                     }
 920                     fit = fit->next;
 921                 }
 922                 if(dp) remove_disk(&tapeq, dp);
 923                 break;
 924         case ALGO_LAST:
 925                 dp = tapeq.tail;
 926                 remove_disk(&tapeq, dp);
 927                 break;
 928         case ALGO_LASTFIT:
 929                 fit = tapeq.tail;
 930                 while (fit != NULL) {
 931                     if (sched(fit)->act_size <=
 932                         (fit->splitsize ? extra_tapes_size : taper_left) &&
 933                         (!dp || sched(fit)->act_size < sched(dp)->act_size) &&
 934                         strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 935                         dp = fit;
 936                     }
 937                     fit = fit->prev;
 938                 }
 939                 if(dp) remove_disk(&tapeq, dp);
 940                 break;
 941         }
 942         if (!dp) {
 943             if (!(result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT)) {
 944                 if(conf_taperalgo != ALGO_SMALLEST)  {
 945                     g_fprintf(stderr,
 946                         _("driver: startaflush: Using SMALLEST because nothing fit\n"));
 947                 }
 948
 949                 fit = dp = tapeq.head;
 950                 while (fit != NULL) {
 951                     if (sched(fit)->act_size < sched(dp)->act_size &&
 952                         strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 953                         dp = fit;
 954                     }
 955                     fit = fit->next;
 956                 }
 957                 if(dp) remove_disk(&tapeq, dp);
 958             }
 959         }
 960         if (dp) {
 961             taper->disk = dp;
 962             taper->dumper = NULL;
 963             amfree(taper->input_error);
 964             amfree(taper->tape_error);
 965             taper->result = LAST_TOK;
 966             taper->sendresult = 0;
 967             amfree(taper->first_label);
 968             taper->written = 0;
 969             taper->state &= ~TAPER_STATE_IDLE;
 970             taper->state |= TAPER_STATE_FILE_TO_TAPE;
 971             taper->dumper = NULL;
 972             qname = quote_string(dp->name);
 973             if (taper_nb_wait_reply == 0) {
 974                 taper_ev_read = event_register(taper_fd, EV_READFD,
 975                                                handle_taper_result, NULL);
 976             }
 977             taper_nb_wait_reply++;
 978             taper->nb_dle++;
 979             sched(dp)->taper = taper;
 980             taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level,
 981                       sched(dp)->datestamp);
 982             g_fprintf(stderr,_("driver: startaflush: %s %s %s %lld %lld\n"),
 983                     taperalgo2str(taperalgo), dp->host->hostname, qname,
 984                     (long long)sched(taper->disk)->act_size,
 985                     (long long)taper->left);
 986             amfree(qname);
 987             *state_changed = TRUE;
 988         }
 989     }
 990 }
 991
 992 static int
 993 client_constrained(
 994     disk_t *    dp)
 995 {
 996     disk_t *dp2;
 997
 998     /* first, check if host is too busy */
 999
1000     if(dp->host->inprogress >= dp->host->maxdumps) {
1001         return 1;
1002     }
1003
1004     /* next, check conflict with other dumps on same spindle */
1005
1006     if(dp->spindle == -1) {     /* but spindle -1 never conflicts by def. */
1007         return 0;
1008     }
1009
1010     for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext)
1011         if(dp2->inprogress && dp2->spindle == dp->spindle) {
1012             return 1;
1013         }
1014
1015     return 0;
1016 }
1017
1018 static void
1019 allow_dump_dle(
1020     disk_t         *diskp,
1021     taper_t        *taper,
1022     char            dumptype,
1023     disklist_t     *rq,
1024     const time_t    now,
1025     int             dumper_to_holding,
1026     int            *cur_idle,
1027     disk_t        **delayed_diskp,
1028     disk_t        **diskp_accept,
1029     assignedhd_t ***holdp_accept,
1030     off_t           extra_tapes_size)
1031 {
1032     assignedhd_t **holdp=NULL;
1033
1034     if (diskp->host->start_t > now) {
1035         *cur_idle = max(*cur_idle, IDLE_START_WAIT);
1036         if (*delayed_diskp == NULL || sleep_time > diskp->host->start_t) {
1037             *delayed_diskp = diskp;
1038             sleep_time = diskp->host->start_t;
1039         }
1040     } else if(diskp->start_t > now) {
1041         *cur_idle = max(*cur_idle, IDLE_START_WAIT);
1042         if (*delayed_diskp == NULL || sleep_time > diskp->start_t) {
1043             *delayed_diskp = diskp;
1044             sleep_time = diskp->start_t;
1045         }
1046     } else if (diskp->host->netif->curusage > 0 &&
1047                sched(diskp)->est_kps > free_kps(diskp->host->netif)) {
1048         *cur_idle = max(*cur_idle, IDLE_NO_BANDWIDTH);
1049     } else if (!taper && sched(diskp)->no_space) {
1050         *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1051     } else if (!taper && diskp->to_holdingdisk == HOLD_NEVER) {
1052         *cur_idle = max(*cur_idle, IDLE_NO_HOLD);
1053     } else if (extra_tapes_size && sched(diskp)->est_size > extra_tapes_size) {
1054         *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1055         /* no tape space */
1056     } else if (!taper && (holdp =
1057         find_diskspace(sched(diskp)->est_size, cur_idle, NULL)) == NULL) {
1058         *cur_idle = max(*cur_idle, IDLE_NO_DISKSPACE);
1059         if (empty(tapeq) && dumper_to_holding == 0 && rq != &directq && no_taper_flushing()) {
1060             remove_disk(rq, diskp);
1061             if (diskp->to_holdingdisk != HOLD_REQUIRED) {
1062                 enqueue_disk(&directq, diskp);
1063                 diskp->to_holdingdisk = HOLD_NEVER;
1064             }
1065             if (empty(*rq) && active_dumper() == 0) { force_flush = 1;}
1066         }
1067     } else if (client_constrained(diskp)) {
1068         free_assignedhd(holdp);
1069         *cur_idle = max(*cur_idle, IDLE_CLIENT_CONSTRAINED);
1070     } else {
1071
1072         /* disk fits, dump it */
1073         int accept = !*diskp_accept;
1074         if(!accept) {
1075             switch(dumptype) {
1076               case 's': accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1077                         break;
1078               case 'S': accept = (sched(diskp)->est_size > sched(*diskp_accept)->est_size);
1079                         break;
1080               case 't': accept = (sched(diskp)->est_time < sched(*diskp_accept)->est_time);
1081                         break;
1082               case 'T': accept = (sched(diskp)->est_time > sched(*diskp_accept)->est_time);
1083                         break;
1084               case 'b': accept = (sched(diskp)->est_kps < sched(*diskp_accept)->est_kps);
1085                         break;
1086               case 'B': accept = (sched(diskp)->est_kps > sched(*diskp_accept)->est_kps);
1087                         break;
1088               default:  log_add(L_WARNING, _("Unknown dumporder character \'%c\', using 's'.\n"),
1089                                 dumptype);
1090                         accept = (sched(diskp)->est_size < sched(*diskp_accept)->est_size);
1091                         break;
1092             }
1093         }
1094         if(accept) {
1095             if( !*diskp_accept || !degraded_mode || diskp->priority >= (*diskp_accept)->priority) {
1096                 if(*holdp_accept) free_assignedhd(*holdp_accept);
1097                 *diskp_accept = diskp;
1098                 *holdp_accept = holdp;
1099             }
1100             else {
1101                 free_assignedhd(holdp);
1102             }
1103         }
1104         else {
1105             free_assignedhd(holdp);
1106         }
1107     }
1108 }
1109
1110 static void
1111 start_some_dumps(
1112     disklist_t *rq)
1113 {
1114     const time_t now = time(NULL);
1115     int cur_idle;
1116     disk_t *diskp, *delayed_diskp, *diskp_accept, *diskp_next;
1117     disk_t *dp;
1118     assignedhd_t **holdp=NULL, **holdp_accept;
1119     cmd_t cmd;
1120     int result_argc;
1121     char **result_argv;
1122     chunker_t *chunker;
1123     dumper_t *dumper;
1124     taper_t  *taper;
1125     char dumptype;
1126     char *dumporder;
1127     int  dumper_to_holding = 0;
1128     gboolean state_changed = FALSE;
1129
1130     /* don't start any actual dumps until the taper is started */
1131     if (!taper_started) return;
1132
1133     idle_reason = IDLE_NO_DUMPERS;
1134     sleep_time = 0;
1135
1136     if(dumpers_ev_time != NULL) {
1137         event_release(dumpers_ev_time);
1138         dumpers_ev_time = NULL;
1139     }
1140
1141     for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1142         if (dumper->busy && dumper->dp->to_holdingdisk != HOLD_NEVER) {
1143             dumper_to_holding++;
1144         }
1145     }
1146     for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) {
1147
1148         if( dumper->busy || dumper->down) {
1149             continue;
1150         }
1151
1152         if (dumper->ev_read != NULL) {
1153             event_release(dumper->ev_read);
1154             dumper->ev_read = NULL;
1155         }
1156
1157         /*
1158          * A potential problem with starting from the bottom of the dump time
1159          * distribution is that a slave host will have both one of the shortest
1160          * and one of the longest disks, so starting its shortest disk first will
1161          * tie up the host and eliminate its longest disk from consideration the
1162          * first pass through.  This could cause a big delay in starting that long
1163          * disk, which could drag out the whole night's dumps.
1164          *
1165          * While starting from the top of the dump time distribution solves the
1166          * above problem, this turns out to be a bad idea, because the big dumps
1167          * will almost certainly pack the holding disk completely, leaving no
1168          * room for even one small dump to start.  This ends up shutting out the
1169          * small-end dumpers completely (they stay idle).
1170          *
1171          * The introduction of multiple simultaneous dumps to one host alleviates
1172          * the biggest&smallest dumps problem: both can be started at the
1173          * beginning.
1174          */
1175
1176         diskp_accept = NULL;
1177         holdp_accept = NULL;
1178         delayed_diskp = NULL;
1179
1180         cur_idle = NOT_IDLE;
1181
1182         dumporder = getconf_str(CNF_DUMPORDER);
1183         if(strlen(dumporder) > (size_t)(dumper-dmptable)) {
1184             dumptype = dumporder[dumper-dmptable];
1185         }
1186         else {
1187             if(dumper-dmptable < 3)
1188                 dumptype = 't';
1189             else
1190                 dumptype = 'T';
1191         }
1192
1193         diskp = NULL;
1194         taper = NULL;
1195         if (!empty(directq)) {
1196             taper = idle_taper();
1197             if (taper) {
1198                 TapeAction result_tape_action;
1199                 char *why_no_new_tape = NULL;
1200                 result_tape_action = tape_action(taper, &why_no_new_tape);
1201                 if (result_tape_action & TAPE_ACTION_START_A_FLUSH ||
1202                     result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1203                     off_t extra_tapes_size = 0;
1204                     taper_t *taper1;
1205
1206                     if (result_tape_action & TAPE_ACTION_START_A_FLUSH_FIT) {
1207                         extra_tapes_size = tape_length *
1208                                           (off_t)(conf_runtapes - current_tape);
1209                         for (taper1 = tapetable;
1210                              taper1 < tapetable + conf_taper_parallel_write;
1211                              taper1++) {
1212                             if (taper1->state & TAPER_STATE_TAPE_STARTED) {
1213                                 extra_tapes_size += taper1->left;
1214                             }
1215                             dp = taper1->disk;
1216                             if (dp) {
1217                                 extra_tapes_size -= (sched(dp)->est_size -
1218                                                     taper1->written);
1219                             }
1220                         }
1221                     }
1222
1223                     for (diskp = directq.head; diskp != NULL;
1224                                                diskp = diskp_next) {
1225                         diskp_next = diskp->next;
1226                         allow_dump_dle(diskp, taper, dumptype, &directq, now,
1227                                        dumper_to_holding, &cur_idle,
1228                                        &delayed_diskp, &diskp_accept,
1229                                        &holdp_accept, extra_tapes_size);
1230                     }
1231                     if (diskp_accept) {
1232                         diskp = diskp_accept;
1233                         holdp = holdp_accept;
1234                     } else {
1235                         taper = NULL;
1236                     }
1237                 } else {
1238                     taper = NULL;
1239                 }
1240             }
1241         }
1242
1243         if (diskp == NULL) {
1244             for(diskp = rq->head; diskp != NULL; diskp = diskp_next) {
1245                 diskp_next = diskp->next;
1246                 assert(diskp->host != NULL && sched(diskp) != NULL);
1247
1248                 allow_dump_dle(diskp, NULL, dumptype, rq, now,
1249                                dumper_to_holding, &cur_idle, &delayed_diskp,
1250                                &diskp_accept, &holdp_accept, 0);
1251             }
1252             diskp = diskp_accept;
1253             holdp = holdp_accept;
1254         }
1255
1256         idle_reason = max(idle_reason, cur_idle);
1257         if (diskp == NULL && idle_reason == IDLE_NO_DISKSPACE) {
1258             /* continue flush waiting for new tape */
1259             startaflush();
1260         }
1261
1262         /*
1263          * If we have no disk at this point, and there are disks that
1264          * are delayed, then schedule a time event to call this dumper
1265          * with the disk with the shortest delay.
1266          */
1267         if (diskp == NULL && delayed_diskp != NULL) {
1268             assert(sleep_time > now);
1269             sleep_time -= now;
1270             dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME,
1271                 handle_dumpers_time, &runq);
1272             return;
1273         } else if (diskp != NULL && taper == NULL) {
1274             sched(diskp)->act_size = (off_t)0;
1275             allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1276             sched(diskp)->activehd = assign_holdingdisk(holdp, diskp);
1277             amfree(holdp);
1278             sched(diskp)->destname = newstralloc(sched(diskp)->destname,
1279                                                  sched(diskp)->holdp[0]->destname);
1280             diskp->host->inprogress++;  /* host is now busy */
1281             diskp->inprogress = 1;
1282             sched(diskp)->dumper = dumper;
1283             sched(diskp)->timestamp = now;
1284             amfree(diskp->dataport_list);
1285
1286             dumper->busy = 1;           /* dumper is now busy */
1287             dumper->dp = diskp;         /* link disk to dumper */
1288             remove_disk(rq, diskp);             /* take it off the run queue */
1289
1290             sched(diskp)->origsize = (off_t)-1;
1291             sched(diskp)->dumpsize = (off_t)-1;
1292             sched(diskp)->dumptime = (time_t)0;
1293             sched(diskp)->tapetime = (time_t)0;
1294             chunker = dumper->chunker = &chktable[dumper - dmptable];
1295             chunker->result = LAST_TOK;
1296             dumper->result = LAST_TOK;
1297             startup_chunk_process(chunker,chunker_program);
1298             chunker_cmd(chunker, START, NULL, driver_timestamp);
1299             chunker->dumper = dumper;
1300             chunker_cmd(chunker, PORT_WRITE, diskp, NULL);
1301             cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
1302             if(cmd != PORT) {
1303                 assignedhd_t **h=NULL;
1304                 int activehd;
1305                 char *qname = quote_string(diskp->name);
1306
1307                 g_printf(_("driver: did not get PORT from %s for %s:%s\n"),
1308                        chunker->name, diskp->host->hostname, qname);
1309                 amfree(qname);
1310                 fflush(stdout);
1311
1312                 deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1313                 h = sched(diskp)->holdp;
1314                 activehd = sched(diskp)->activehd;
1315                 h[activehd]->used = 0;
1316                 h[activehd]->disk->allocated_dumpers--;
1317                 adjust_diskspace(diskp, DONE);
1318                 delete_diskspace(diskp);
1319                 diskp->host->inprogress--;
1320                 diskp->inprogress = 0;
1321                 sched(diskp)->dumper = NULL;
1322                 dumper->busy = 0;
1323                 dumper->dp = NULL;
1324                 sched(diskp)->dump_attempted++;
1325                 free_serial_dp(diskp);
1326                 if(sched(diskp)->dump_attempted < 2)
1327                     enqueue_disk(rq, diskp);
1328             }
1329             else {
1330                 dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD,
1331                                                  handle_dumper_result, dumper);
1332                 chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD,
1333                                                    handle_chunker_result, chunker);
1334                 dumper->output_port = atoi(result_argv[1]);
1335                 amfree(diskp->dataport_list);
1336                 diskp->dataport_list = stralloc(result_argv[2]);
1337
1338                 if (diskp->host->pre_script == 0) {
1339                     run_server_host_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1340                                             get_config_name(), diskp->host);
1341                     diskp->host->pre_script = 1;
1342                 }
1343                 run_server_dle_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1344                                    get_config_name(), diskp,
1345                                    sched(diskp)->level);
1346                 dumper_cmd(dumper, PORT_DUMP, diskp, NULL);
1347             }
1348             diskp->host->start_t = now + 5;
1349             if (empty(*rq) && active_dumper() == 0) { force_flush = 1;}
1350
1351             if (result_argv)
1352                 g_strfreev(result_argv);
1353             short_dump_state();
1354         } else if (diskp != NULL && taper != NULL) { /* dump to tape */
1355             sched(diskp)->act_size = (off_t)0;
1356             allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
1357             diskp->host->inprogress++;  /* host is now busy */
1358             diskp->inprogress = 1;
1359             sched(diskp)->dumper = dumper;
1360             sched(diskp)->taper = taper;
1361             sched(diskp)->timestamp = now;
1362             dumper->chunker = NULL;
1363             amfree(diskp->dataport_list);
1364
1365             dumper->busy = 1;           /* dumper is now busy */
1366             dumper->dp = diskp;         /* link disk to dumper */
1367             remove_disk(&directq, diskp);  /* take it off the direct queue */
1368
1369             sched(diskp)->origsize = (off_t)-1;
1370             sched(diskp)->dumpsize = (off_t)-1;
1371             sched(diskp)->dumptime = (time_t)0;
1372             sched(diskp)->tapetime = (time_t)0;
1373             dumper->result = LAST_TOK;
1374             taper->result = LAST_TOK;
1375             taper->input_error = NULL;
1376             taper->tape_error = NULL;
1377             taper->disk = diskp;
1378             taper->first_label = NULL;
1379             taper->written = 0;
1380             taper->dumper = dumper;
1381             taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1382             taper->state &= ~TAPER_STATE_IDLE;
1383             taper->nb_dle++;
1384             if (taper_nb_wait_reply == 0) {
1385                 taper_ev_read = event_register(taper_fd, EV_READFD,
1386                                                handle_taper_result, NULL);
1387             }
1388
1389             taper_nb_wait_reply++;
1390             taper_cmd(PORT_WRITE, diskp, NULL, sched(diskp)->level,
1391                       sched(diskp)->datestamp);
1392             diskp->host->start_t = now + 5;
1393
1394             state_changed = TRUE;
1395         }
1396     }
1397     if (state_changed) {
1398         short_dump_state();
1399     }
1400 }
1401
1402 /*
1403  * This gets called when a dumper is delayed for some reason.  It may
1404  * be because a disk has a delayed start, or amanda is constrained
1405  * by network or disk limits.
1406  */
1407
1408 static void
1409 handle_dumpers_time(
1410     void *      cookie)
1411 {
1412     disklist_t *runq = cookie;
1413     event_release(dumpers_ev_time);
1414     dumpers_ev_time = NULL;
1415     start_some_dumps(runq);
1416 }
1417
1418 static void
1419 dump_schedule(
1420     disklist_t *qp,
1421     char *      str)
1422 {
1423     disk_t *dp;
1424     char *qname;
1425
1426     g_printf(_("dump of driver schedule %s:\n--------\n"), str);
1427
1428     for(dp = qp->head; dp != NULL; dp = dp->next) {
1429         qname = quote_string(dp->name);
1430         g_printf("  %-20s %-25s lv %d t %5lu s %lld p %d\n",
1431                dp->host->hostname, qname, sched(dp)->level,
1432                sched(dp)->est_time,
1433                (long long)sched(dp)->est_size, sched(dp)->priority);
1434         amfree(qname);
1435     }
1436     g_printf("--------\n");
1437 }
1438
1439 static void
1440 start_degraded_mode(
1441     /*@keep@*/ disklist_t *queuep)
1442 {
1443     disk_t *dp;
1444     disklist_t newq;
1445     off_t est_full_size;
1446     char *qname;
1447     taper_t *taper;
1448
1449     if (need_degraded == 0) {
1450         for(taper = tapetable;
1451             taper < tapetable+conf_taper_parallel_write;
1452             taper++) {
1453             if (!(taper->state & TAPER_STATE_DONE))
1454                 return;
1455         }
1456         need_degraded = 1;
1457     }
1458
1459     if (!schedule_done || degraded_mode) {
1460         return;
1461     }
1462
1463     if (need_degraded == 0) {
1464         for(taper = tapetable;
1465             taper < tapetable+conf_taper_parallel_write;
1466             taper++) {
1467             if (!(taper->state & TAPER_STATE_DONE))
1468                 return;
1469         }
1470         need_degraded = 1;
1471     }
1472
1473     newq.head = newq.tail = 0;
1474
1475     dump_schedule(queuep, _("before start degraded mode"));
1476
1477     est_full_size = (off_t)0;
1478     while(!empty(*queuep)) {
1479         dp = dequeue_disk(queuep);
1480
1481         qname = quote_string(dp->name);
1482         if(sched(dp)->level != 0)
1483             /* go ahead and do the disk as-is */
1484             enqueue_disk(&newq, dp);
1485         else {
1486             if (reserved_space + est_full_size + sched(dp)->est_size
1487                 <= total_disksize) {
1488                 enqueue_disk(&newq, dp);
1489                 est_full_size += sched(dp)->est_size;
1490             }
1491             else if(sched(dp)->degr_level != -1) {
1492                 sched(dp)->level = sched(dp)->degr_level;
1493                 sched(dp)->dumpdate = sched(dp)->degr_dumpdate;
1494                 sched(dp)->est_nsize = sched(dp)->degr_nsize;
1495                 sched(dp)->est_csize = sched(dp)->degr_csize;
1496                 sched(dp)->est_time = sched(dp)->degr_time;
1497                 sched(dp)->est_kps  = sched(dp)->degr_kps;
1498                 enqueue_disk(&newq, dp);
1499             }
1500             else {
1501                 log_add(L_FAIL, "%s %s %s %d [%s]",
1502                         dp->host->hostname, qname, sched(dp)->datestamp,
1503                         sched(dp)->level, sched(dp)->degr_mesg);
1504             }
1505         }
1506         amfree(qname);
1507     }
1508
1509     /*@i@*/ *queuep = newq;
1510     degraded_mode = 1;
1511
1512     dump_schedule(queuep, _("after start degraded mode"));
1513 }
1514
1515
1516 static void
1517 continue_port_dumps(void)
1518 {
1519     disk_t *dp, *ndp;
1520     assignedhd_t **h;
1521     int active_dumpers=0, busy_dumpers=0, i;
1522     dumper_t *dumper;
1523
1524     /* First we try to grant diskspace to some dumps waiting for it. */
1525     for( dp = roomq.head; dp; dp = ndp ) {
1526         ndp = dp->next;
1527         /* find last holdingdisk used by this dump */
1528         for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) {
1529             (void)h; /* Quiet lint */
1530         }
1531         /* find more space */
1532         h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1533                             &active_dumpers, h[i] );
1534         if( h ) {
1535             for(dumper = dmptable; dumper < dmptable + inparallel &&
1536                                    dumper->dp != dp; dumper++) {
1537                 (void)dp; /* Quiet lint */
1538             }
1539             assert( dumper < dmptable + inparallel );
1540             sched(dp)->activehd = assign_holdingdisk( h, dp );
1541             chunker_cmd( dumper->chunker, CONTINUE, dp, NULL );
1542             amfree(h);
1543             remove_disk( &roomq, dp );
1544         }
1545     }
1546
1547     /* So for some disks there is less holding diskspace available than
1548      * was asked for. Possible reasons are
1549      * a) diskspace has been allocated for other dumps which are
1550      *    still running or already being written to tape
1551      * b) all other dumps have been suspended due to lack of diskspace
1552      * Case a) is not a problem. We just wait for the diskspace to
1553      * be freed by moving the current disk to a queue.
1554      * If case b) occurs, we have a deadlock situation. We select
1555      * a dump from the queue to be aborted and abort it. It will
1556      * be retried directly to tape.
1557      */
1558     for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1559         if( dumper->busy ) {
1560             busy_dumpers++;
1561             if( !find_disk(&roomq, dumper->dp) ) {
1562                 if (dumper->chunker) {
1563                     active_dumpers++;
1564                 }
1565             } else if( !dp ||
1566                        sched(dp)->est_size > sched(dumper->dp)->est_size ) {
1567                 dp = dumper->dp;
1568             }
1569         }
1570     }
1571     if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) &&
1572         ((no_taper_flushing() && empty(tapeq)) || degraded_mode) &&
1573         pending_aborts == 0 ) { /* case b */
1574         sched(dp)->no_space = 1;
1575         /* At this time, dp points to the dump with the smallest est_size.
1576          * We abort that dump, hopefully not wasting too much time retrying it.
1577          */
1578         remove_disk( &roomq, dp );
1579         chunker_cmd(sched(dp)->dumper->chunker, ABORT, NULL, _("Not enough holding disk space"));
1580         dumper_cmd( sched(dp)->dumper, ABORT, NULL, _("Not enough holding disk space"));
1581         pending_aborts++;
1582     }
1583 }
1584
1585
1586 static void
1587 handle_taper_result(
1588         void *cookie G_GNUC_UNUSED)
1589 {
1590     disk_t *dp = NULL;
1591     dumper_t *dumper;
1592     cmd_t cmd;
1593     int result_argc;
1594     char **result_argv;
1595     char *qname, *q;
1596     char *s;
1597     taper_t *taper = NULL;
1598     taper_t *taper1;
1599     int      i;
1600     off_t    partsize;
1601
1602     assert(cookie == NULL);
1603
1604     do {
1605
1606         short_dump_state();
1607         taper = NULL;
1608
1609         cmd = getresult(taper_fd, 1, &result_argc, &result_argv);
1610
1611         switch(cmd) {
1612
1613         case TAPER_OK:
1614             if(result_argc != 2) {
1615                 error(_("error: [taper FAILED result_argc != 2: %d"), result_argc);
1616                 /*NOTREACHED*/
1617             }
1618
1619             taper = NULL;
1620             taper_started = 1;
1621             for (i=0; i < conf_taper_parallel_write; i++) {
1622                 if (strcmp(tapetable[i].name, result_argv[1]) == 0) {
1623                     taper= &tapetable[i];
1624                 }
1625             }
1626             assert(taper != NULL);
1627             taper->left = 0;
1628             taper->nb_dle = 0;
1629             taper->state &= ~TAPER_STATE_INIT;
1630             taper->state |= TAPER_STATE_RESERVATION;
1631             taper->state |= TAPER_STATE_IDLE;
1632             amfree(taper->first_label);
1633             taper_nb_wait_reply--;
1634             taper_nb_scan_volume--;
1635             last_started_taper = taper;
1636             if (taper_nb_wait_reply == 0) {
1637                 event_release(taper_ev_read);
1638                 taper_ev_read = NULL;
1639             }
1640             start_some_dumps(&runq);
1641             startaflush();
1642             break;
1643
1644         case FAILED:    /* FAILED <handle> INPUT-* TAPE-* <input err mesg> <tape err mesg> */
1645             if(result_argc != 6) {
1646                 error(_("error: [taper FAILED result_argc != 6: %d"), result_argc);
1647                 /*NOTREACHED*/
1648             }
1649
1650             dp = serial2disk(result_argv[1]);
1651             taper = sched(dp)->taper;
1652             assert(dp == taper->disk);
1653             if (!taper->dumper)
1654                 free_serial(result_argv[1]);
1655
1656             qname = quote_string(dp->name);
1657             g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1658                    walltime_str(curclock()), dp->host->hostname, qname);
1659             fflush(stdout);
1660
1661             if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1662                 taper->input_error = newstralloc(taper->input_error, result_argv[4]);
1663                 taper->result = FAILED;
1664                 amfree(qname);
1665                 break;
1666             } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1667                 taper->tape_error = newstralloc(taper->tape_error,
1668                                                _("Taper protocol error"));
1669                 taper->result = FAILED;
1670                 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1671                         dp->host->hostname, qname, sched(dp)->datestamp,
1672                         sched(dp)->level, taper->tape_error);
1673                 amfree(qname);
1674                 break;
1675             }
1676             if (strcmp(result_argv[3], "TAPE-ERROR") == 0 ||
1677                 strcmp(result_argv[3], "TAPE-CONFIG") == 0) {
1678                 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1679                 taper->tape_error = newstralloc(taper->tape_error, result_argv[5]);
1680                 taper->result = FAILED;
1681                 amfree(qname);
1682                 break;
1683             } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1684                 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1685                 taper->tape_error = newstralloc(taper->tape_error,
1686                                                _("Taper protocol error"));
1687                 taper->result = FAILED;
1688                 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1689                         dp->host->hostname, qname, sched(dp)->datestamp,
1690                         sched(dp)->level, taper->tape_error);
1691                 amfree(qname);
1692                 break;
1693             }
1694
1695             amfree(qname);
1696             taper->result = cmd;
1697
1698             break;
1699
1700         case PARTIAL:   /* PARTIAL <handle> INPUT-* TAPE-* <stat mess> <input err mesg> <tape err mesg>*/
1701         case DONE:      /* DONE <handle> INPUT-GOOD TAPE-GOOD <stat mess> <input err mesg> <tape err mesg> */
1702             if(result_argc != 7) {
1703                 error(_("error: [taper PARTIAL result_argc != 7: %d"), result_argc);
1704                 /*NOTREACHED*/
1705             }
1706
1707             dp = serial2disk(result_argv[1]);
1708             taper = sched(dp)->taper;
1709             assert(dp == taper->disk);
1710             if (!taper->dumper)
1711                 free_serial(result_argv[1]);
1712
1713             qname = quote_string(dp->name);
1714             g_printf(_("driver: finished-cmd time %s taper wrote %s:%s\n"),
1715                    walltime_str(curclock()), dp->host->hostname, qname);
1716             fflush(stdout);
1717
1718             if (strcmp(result_argv[2], "INPUT-ERROR") == 0) {
1719                 taper->input_error = newstralloc(taper->input_error, result_argv[5]);
1720                 taper->result = FAILED;
1721                 amfree(qname);
1722                 break;
1723             } else if (strcmp(result_argv[2], "INPUT-GOOD") != 0) {
1724                 taper->tape_error = newstralloc(taper->tape_error,
1725                                                _("Taper protocol error"));
1726                 taper->result = FAILED;
1727                 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1728                         dp->host->hostname, qname, sched(dp)->datestamp,
1729                         sched(dp)->level, taper->tape_error);
1730                 amfree(qname);
1731                 break;
1732             }
1733             if (strcmp(result_argv[3], "TAPE-ERROR") == 0 ||
1734                 strcmp(result_argv[3], "TAPE-CONFIG") == 0) {
1735                 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1736                 taper->tape_error = newstralloc(taper->tape_error, result_argv[6]);
1737                 taper->result = FAILED;
1738                 amfree(qname);
1739                 break;
1740             } else if (strcmp(result_argv[3], "TAPE-GOOD") != 0) {
1741                 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1742                 taper->tape_error = newstralloc(taper->tape_error,
1743                                                _("Taper protocol error"));
1744                 taper->result = FAILED;
1745                 log_add(L_FAIL, _("%s %s %s %d [%s]"),
1746                         dp->host->hostname, qname, sched(dp)->datestamp,
1747                         sched(dp)->level, taper->tape_error);
1748                 amfree(qname);
1749                 break;
1750             }
1751
1752             s = strstr(result_argv[4], " kb ");
1753             if (s) {
1754                 s += 4;
1755                 sched(dp)->dumpsize = OFF_T_ATOI(s);
1756             } else {
1757                 s = strstr(result_argv[4], " bytes ");
1758                 if (s) {
1759                     s += 7;
1760                     sched(dp)->dumpsize = OFF_T_ATOI(s)/1024;
1761                 }
1762             }
1763
1764             taper->result = cmd;
1765             amfree(qname);
1766
1767             break;
1768
1769         case PARTDONE:  /* PARTDONE <handle> <label> <fileno> <kbytes> <stat> */
1770             dp = serial2disk(result_argv[1]);
1771             taper = sched(dp)->taper;
1772             assert(dp == taper->disk);
1773             if (result_argc != 6) {
1774                 error(_("error [taper PARTDONE result_argc != 6: %d]"),
1775                       result_argc);
1776                 /*NOTREACHED*/
1777             }
1778             if (!taper->first_label) {
1779                 amfree(taper->first_label);
1780                 taper->first_label = stralloc(result_argv[2]);
1781                 taper->first_fileno = OFF_T_ATOI(result_argv[3]);
1782             }
1783             taper->written += OFF_T_ATOI(result_argv[4]);
1784             if (taper->written > sched(taper->disk)->act_size)
1785                 sched(taper->disk)->act_size = taper->written;
1786
1787             partsize = 0;
1788             s = strstr(result_argv[5], " kb ");
1789             if (s) {
1790                 s += 4;
1791                 partsize = OFF_T_ATOI(s);
1792             } else {
1793                 s = strstr(result_argv[5], " bytes ");
1794                 if (s) {
1795                     s += 7;
1796                     partsize = OFF_T_ATOI(s)/1024;
1797                 }
1798             }
1799             taper->left -= partsize;
1800
1801             break;
1802
1803         case REQUEST_NEW_TAPE:  /* REQUEST-NEW-TAPE <handle> */
1804             if (result_argc != 2) {
1805                 error(_("error [taper REQUEST_NEW_TAPE result_argc != 2: %d]"),
1806                       result_argc);
1807                 /*NOTREACHED*/
1808             }
1809
1810             dp = serial2disk(result_argv[1]);
1811             taper = sched(dp)->taper;
1812             if (taper->state & TAPER_STATE_DONE) {
1813                 taper_cmd(NO_NEW_TAPE, taper->disk, "taper found no tape", 0, NULL);
1814             } else {
1815                 taper->state &= ~TAPER_STATE_TAPE_STARTED;
1816                 taper->state |= TAPER_STATE_TAPE_REQUESTED;
1817
1818                 start_some_dumps(&runq);
1819                 startaflush();
1820             }
1821             break;
1822
1823         case NEW_TAPE: /* NEW-TAPE <handle> <label> */
1824             if (result_argc != 3) {
1825                 error(_("error [taper NEW_TAPE result_argc != 3: %d]"),
1826                       result_argc);
1827                 /*NOTREACHED*/
1828             }
1829
1830             nb_sent_new_tape--;
1831             taper_nb_scan_volume--;
1832             dp = serial2disk(result_argv[1]);
1833             taper = sched(dp)->taper;
1834             /* Update our tape counter and reset taper->left */
1835             current_tape++;
1836             taper->nb_dle = 1;
1837             taper->left = tape_length;
1838             taper->state &= ~TAPER_STATE_WAIT_NEW_TAPE;
1839             taper->state |= TAPER_STATE_TAPE_STARTED;
1840             last_started_taper = NULL;
1841
1842             /* start a new worker */
1843             for (i = 0; i < conf_taper_parallel_write ; i++) {
1844                 taper1 = &tapetable[i];
1845                 if (need_degraded == 0 &&
1846                     taper1->state == TAPER_STATE_DEFAULT) {
1847                     taper1->state = TAPER_STATE_INIT;
1848                     if (taper_nb_wait_reply == 0) {
1849                         taper_ev_read = event_register(taper_fd, EV_READFD,
1850                                                 handle_taper_result, NULL);
1851                     }
1852                     taper_nb_wait_reply++;
1853                     taper_nb_scan_volume++;
1854                     taper_cmd(START_TAPER, NULL, taper1->name, 0,
1855                               driver_timestamp);
1856                     break;
1857                 }
1858             }
1859             break;
1860
1861         case NO_NEW_TAPE:  /* NO-NEW-TAPE <handle> */
1862             if (result_argc != 2) {
1863                 error(_("error [taper NO_NEW_TAPE result_argc != 2: %d]"),
1864                       result_argc);
1865                 /*NOTREACHED*/
1866             }
1867             nb_sent_new_tape--;
1868             taper_nb_scan_volume--;
1869             dp = serial2disk(result_argv[1]);
1870             taper = sched(dp)->taper;
1871             taper->state |= TAPER_STATE_DONE;
1872             last_started_taper = NULL;
1873             start_degraded_mode(&runq);
1874             break;
1875
1876         case DUMPER_STATUS:  /* DUMPER-STATUS <handle> */
1877             if (result_argc != 2) {
1878                 error(_("error [taper DUMPER_STATUS result_argc != 2: %d]"),
1879                       result_argc);
1880                 /*NOTREACHED*/
1881             }
1882             dp = serial2disk(result_argv[1]);
1883             taper = sched(dp)->taper;
1884             if (taper->dumper->result == LAST_TOK) {
1885                 taper->sendresult = 1;
1886             } else {
1887                 if( taper->dumper->result == DONE) {
1888                     taper_cmd(DONE, dp, NULL, 0, NULL);
1889                 } else {
1890                     taper_cmd(FAILED, dp, NULL, 0, NULL);
1891                 }
1892             }
1893             break;
1894
1895         case TAPE_ERROR: /* TAPE-ERROR <name> <err mess> */
1896             taper_started = 1;
1897             if (strcmp(result_argv[1], "SETUP") == 0) {
1898                 taper_nb_wait_reply = 0;
1899                 taper_nb_scan_volume = 0;
1900                 need_degraded = 1;
1901             } else {
1902                 taper = taper_from_name(result_argv[1]);
1903                 taper->state = TAPER_STATE_DONE;
1904                 fflush(stdout);
1905                 q = quote_string(result_argv[2]);
1906                 log_add(L_WARNING, _("Taper error: %s"), q);
1907                 amfree(q);
1908                 if (taper) {
1909                     taper->tape_error = newstralloc(taper->tape_error,
1910                                                     result_argv[2]);
1911                 }
1912
1913                 taper_nb_wait_reply--;
1914                 taper_nb_scan_volume--;
1915             }
1916             if (taper_nb_wait_reply == 0) {
1917                 need_degraded = 1;
1918                 event_release(taper_ev_read);
1919                 taper_ev_read = NULL;
1920             }
1921             start_degraded_mode(&runq);
1922             start_some_dumps(&runq);
1923             break;
1924
1925         case PORT: /* PORT <name> <handle> <port> <dataport_list> */
1926             dp = serial2disk(result_argv[2]);
1927             taper = sched(dp)->taper;
1928             dumper = sched(dp)->dumper;
1929             dumper->output_port = atoi(result_argv[3]);
1930             amfree(dp->dataport_list);
1931             dp->dataport_list = stralloc(result_argv[4]);
1932
1933             amfree(taper->input_error);
1934             amfree(taper->tape_error);
1935             amfree(taper->first_label);
1936             taper->written = 0;
1937             taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1938
1939             if (dp->host->pre_script == 0) {
1940                 run_server_host_scripts(EXECUTE_ON_PRE_HOST_BACKUP,
1941                                         get_config_name(), dp->host);
1942                 dp->host->pre_script = 1;
1943             }
1944             run_server_dle_scripts(EXECUTE_ON_PRE_DLE_BACKUP,
1945                                get_config_name(), dp,
1946                                sched(dp)->level);
1947             /* tell the dumper to dump to a port */
1948             dumper_cmd(dumper, PORT_DUMP, dp, NULL);
1949             dp->host->start_t = time(NULL) + 5;
1950             amfree(dp->dataport_list);
1951
1952             taper->state |= TAPER_STATE_DUMP_TO_TAPE;
1953
1954             dumper->ev_read = event_register(dumper->fd, EV_READFD,
1955                                              handle_dumper_result, dumper);
1956             break;
1957
1958         case BOGUS:
1959             log_add(L_WARNING, _("Taper protocol error"));
1960             /*
1961              * Since we received a taper error, we can't send anything more
1962              * to the taper.  Go into degraded mode to try to get everthing
1963              * onto disk.  Later, these dumps can be flushed to a new tape.
1964              * The tape queue is zapped so that it appears empty in future
1965              * checks. If there are dumps waiting for diskspace to be freed,
1966              * cancel one.
1967              */
1968             taper_started = 1;
1969             if(!nodump) {
1970                 log_add(L_WARNING,
1971                         _("going into degraded mode because of taper component error."));
1972             }
1973
1974             for (taper = tapetable;
1975                  taper < tapetable + conf_taper_parallel_write;
1976                  taper++) {
1977                 if (taper && taper->disk) {
1978                     taper->tape_error = newstralloc(taper->tape_error,"BOGUS");
1979                     taper->result = cmd;
1980                     if (taper->dumper) {
1981                         if (taper->dumper->result != LAST_TOK) {
1982                             // Dumper already returned it's result
1983                             dumper_taper_result(taper->disk);
1984                         }
1985                     } else {
1986                         file_taper_result(taper->disk);
1987                     }
1988                 }
1989             }
1990             taper = NULL;
1991
1992             if(taper_ev_read != NULL) {
1993                 event_release(taper_ev_read);
1994                 taper_ev_read = NULL;
1995                 taper_nb_wait_reply = 0;
1996             }
1997             need_degraded = 1;
1998             start_degraded_mode(&runq);
1999             tapeq.head = tapeq.tail = NULL;
2000             aclose(taper_fd);
2001
2002             break;
2003
2004         default:
2005             error(_("driver received unexpected token (%s) from taper"),
2006                   cmdstr[cmd]);
2007             /*NOTREACHED*/
2008         }
2009
2010         g_strfreev(result_argv);
2011
2012         if (taper && taper->disk && taper->result != LAST_TOK) {
2013             if (taper->nb_dle >= conf_max_dle_by_volume) {
2014                 taper_cmd(CLOSE_VOLUME, dp, NULL, 0, NULL);
2015             }
2016             if(taper->dumper) {
2017                 if (taper->dumper->result != LAST_TOK) {
2018                     // Dumper already returned it's result
2019                     dumper_taper_result(taper->disk);
2020                 }
2021             } else {
2022                 file_taper_result(taper->disk);
2023             }
2024         }
2025
2026     } while(areads_dataready(taper_fd));
2027     start_some_dumps(&runq);
2028     startaflush();
2029 }
2030
2031
2032 static void
2033 file_taper_result(
2034     disk_t *dp)
2035 {
2036     taper_t *taper;
2037     char *qname = quote_string(dp->name);
2038
2039     taper = sched(dp)->taper;
2040     if (taper->result == DONE) {
2041         update_info_taper(dp, taper->first_label, taper->first_fileno,
2042                           sched(dp)->level);
2043     }
2044
2045     sched(dp)->taper_attempted += 1;
2046
2047     if (taper->input_error) {
2048         g_printf("driver: taper failed %s %s: %s\n",
2049                    dp->host->hostname, qname, taper->input_error);
2050         if (strcmp(sched(dp)->datestamp, driver_timestamp) == 0) {
2051             if(sched(dp)->taper_attempted >= 2) {
2052                 log_add(L_FAIL, _("%s %s %s %d [too many taper retries after holding disk error: %s]"),
2053                     dp->host->hostname, qname, sched(dp)->datestamp,
2054                     sched(dp)->level, taper->input_error);
2055                 g_printf("driver: taper failed %s %s, too many taper retry after holding disk error\n",
2056                    dp->host->hostname, qname);
2057                 amfree(sched(dp)->destname);
2058                 amfree(sched(dp)->dumpdate);
2059                 amfree(sched(dp)->degr_dumpdate);
2060                 amfree(sched(dp)->degr_mesg);
2061                 amfree(sched(dp)->datestamp);
2062                 amfree(dp->up);
2063             } else {
2064                 log_add(L_INFO, _("%s %s %s %d [Will retry dump because of holding disk error: %s]"),
2065                         dp->host->hostname, qname, sched(dp)->datestamp,
2066                         sched(dp)->level, taper->input_error);
2067                 g_printf("driver: taper will retry %s %s because of holding disk error\n",
2068                         dp->host->hostname, qname);
2069                 if (dp->to_holdingdisk != HOLD_REQUIRED) {
2070                     dp->to_holdingdisk = HOLD_NEVER;
2071                     sched(dp)->dump_attempted -= 1;
2072                     headqueue_disk(&directq, dp);
2073                 } else {
2074                     amfree(sched(dp)->destname);
2075                     amfree(sched(dp)->dumpdate);
2076                     amfree(sched(dp)->degr_dumpdate);
2077                     amfree(sched(dp)->degr_mesg);
2078                     amfree(sched(dp)->datestamp);
2079                     amfree(dp->up);
2080                 }
2081             }
2082         } else {
2083             amfree(sched(dp)->destname);
2084             amfree(sched(dp)->dumpdate);
2085             amfree(sched(dp)->degr_dumpdate);
2086             amfree(sched(dp)->degr_mesg);
2087             amfree(sched(dp)->datestamp);
2088             amfree(dp->up);
2089         }
2090     } else if (taper->tape_error) {
2091         g_printf("driver: taper failed %s %s with tape error: %s\n",
2092                    dp->host->hostname, qname, taper->tape_error);
2093         if(sched(dp)->taper_attempted >= 2) {
2094             log_add(L_FAIL, _("%s %s %s %d [too many taper retries]"),
2095                     dp->host->hostname, qname, sched(dp)->datestamp,
2096                     sched(dp)->level);
2097             g_printf("driver: taper failed %s %s, too many taper retry\n",
2098                    dp->host->hostname, qname);
2099             amfree(sched(dp)->destname);
2100             amfree(sched(dp)->dumpdate);
2101             amfree(sched(dp)->degr_dumpdate);
2102             amfree(sched(dp)->degr_mesg);
2103             amfree(sched(dp)->datestamp);
2104             amfree(dp->up);
2105         } else {
2106             g_printf("driver: taper will retry %s %s\n",
2107                    dp->host->hostname, qname);
2108             /* Re-insert into taper queue. */
2109             headqueue_disk(&tapeq, dp);
2110         }
2111     } else if (taper->result != DONE) {
2112         g_printf("driver: taper failed %s %s without error\n",
2113                    dp->host->hostname, qname);
2114     } else {
2115         delete_diskspace(dp);
2116         amfree(sched(dp)->destname);
2117         amfree(sched(dp)->dumpdate);
2118         amfree(sched(dp)->degr_dumpdate);
2119         amfree(sched(dp)->degr_mesg);
2120         amfree(sched(dp)->datestamp);
2121         amfree(dp->up);
2122     }
2123
2124     amfree(qname);
2125
2126     taper->state &= ~TAPER_STATE_FILE_TO_TAPE;
2127     taper->state |= TAPER_STATE_IDLE;
2128     amfree(taper->input_error);
2129     amfree(taper->tape_error);
2130     taper->disk = NULL;
2131     taper_nb_wait_reply--;
2132     if (taper_nb_wait_reply == 0) {
2133         event_release(taper_ev_read);
2134         taper_ev_read = NULL;
2135     }
2136
2137     /* continue with those dumps waiting for diskspace */
2138     continue_port_dumps();
2139     start_some_dumps(&runq);
2140     startaflush();
2141 }
2142
2143 static void
2144 dumper_taper_result(
2145     disk_t *dp)
2146 {
2147     dumper_t *dumper;
2148     taper_t  *taper;
2149     char *qname;
2150
2151     dumper = sched(dp)->dumper;
2152     taper  = sched(dp)->taper;
2153
2154     free_serial_dp(dp);
2155     if(dumper->result == DONE && taper->result == DONE) {
2156         update_info_dumper(dp, sched(dp)->origsize,
2157                            sched(dp)->dumpsize, sched(dp)->dumptime);
2158         update_info_taper(dp, taper->first_label, taper->first_fileno,
2159                           sched(dp)->level);
2160         qname = quote_string(dp->name); /*quote to take care of spaces*/
2161
2162         log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2163                 dp->host->hostname, qname, sched(dp)->datestamp,
2164                 sched(dp)->level,
2165                 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2166                 (long long)sched(dp)->est_csize,
2167                 sched(dp)->est_kps);
2168         amfree(qname);
2169     } else {
2170         update_failed_dump(dp);
2171     }
2172
2173     sched(dp)->dump_attempted += 1;
2174     sched(dp)->taper_attempted += 1;
2175
2176     if((dumper->result != DONE || taper->result != DONE) &&
2177         sched(dp)->dump_attempted <= 1 &&
2178         sched(dp)->taper_attempted <= 1) {
2179         enqueue_disk(&directq, dp);
2180     }
2181
2182     if(dumper->ev_read != NULL) {
2183         event_release(dumper->ev_read);
2184         dumper->ev_read = NULL;
2185     }
2186     taper_nb_wait_reply--;
2187     if (taper_nb_wait_reply == 0 && taper_ev_read != NULL) {
2188         event_release(taper_ev_read);
2189         taper_ev_read = NULL;
2190     }
2191     taper->state &= ~TAPER_STATE_DUMP_TO_TAPE;
2192     taper->state |= TAPER_STATE_IDLE;
2193     amfree(taper->input_error);
2194     amfree(taper->tape_error);
2195     dumper->busy = 0;
2196     dp->host->inprogress -= 1;
2197     dp->inprogress = 0;
2198     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2199     taper->dumper = NULL;
2200     taper->disk = NULL;
2201     sched(dp)->dumper = NULL;
2202     sched(dp)->taper = NULL;
2203     start_some_dumps(&runq);
2204 }
2205
2206
2207 static taper_t *
2208 idle_taper(void)
2209 {
2210     taper_t *taper;
2211
2212     /* Use an already started taper first */
2213     for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2214                             taper++) {
2215         if ((taper->state & TAPER_STATE_IDLE) &&
2216             (taper->state & TAPER_STATE_TAPE_STARTED) &&
2217             !(taper->state & TAPER_STATE_DONE) &&
2218             !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2219             !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2220             return taper;
2221     }
2222     for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
2223                             taper++) {
2224         if ((taper->state & TAPER_STATE_IDLE) &&
2225             (taper->state & TAPER_STATE_RESERVATION) &&
2226             !(taper->state & TAPER_STATE_DONE) &&
2227             !(taper->state & TAPER_STATE_FILE_TO_TAPE) &&
2228             !(taper->state & TAPER_STATE_FILE_TO_TAPE))
2229             return taper;
2230     }
2231     return NULL;
2232 }
2233
2234 static taper_t *
2235 taper_from_name(
2236     char *name)
2237 {
2238     taper_t *taper;
2239
2240     for (taper = tapetable; taper < tapetable+conf_taper_parallel_write;
2241                             taper++)
2242         if (strcmp(taper->name, name) == 0) return taper;
2243
2244     return NULL;
2245 }
2246
2247 static void
2248 dumper_chunker_result(
2249     disk_t *    dp)
2250 {
2251     dumper_t *dumper;
2252     chunker_t *chunker;
2253     assignedhd_t **h=NULL;
2254     int activehd, i;
2255     off_t dummy;
2256     off_t size;
2257     int is_partial;
2258     char *qname;
2259
2260     dumper = sched(dp)->dumper;
2261     chunker = dumper->chunker;
2262
2263     free_serial_dp(dp);
2264
2265     h = sched(dp)->holdp;
2266     activehd = sched(dp)->activehd;
2267
2268     if(dumper->result == DONE && chunker->result == DONE) {
2269         update_info_dumper(dp, sched(dp)->origsize,
2270                            sched(dp)->dumpsize, sched(dp)->dumptime);
2271         qname = quote_string(dp->name);/*quote to take care of spaces*/
2272
2273         log_add(L_STATS, _("estimate %s %s %s %d [sec %ld nkb %lld ckb %lld kps %lu]"),
2274                 dp->host->hostname, qname, sched(dp)->datestamp,
2275                 sched(dp)->level,
2276                 sched(dp)->est_time, (long long)sched(dp)->est_nsize,
2277                 (long long)sched(dp)->est_csize,
2278                 sched(dp)->est_kps);
2279         amfree(qname);
2280     } else {
2281         update_failed_dump(dp);
2282     }
2283
2284     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2285
2286     is_partial = dumper->result != DONE || chunker->result != DONE;
2287     rename_tmp_holding(sched(dp)->destname, !is_partial);
2288     holding_set_origsize(sched(dp)->destname, sched(dp)->origsize);
2289
2290     dummy = (off_t)0;
2291     for( i = 0, h = sched(dp)->holdp; i < activehd; i++ ) {
2292         dummy += h[i]->used;
2293     }
2294
2295     size = holding_file_size(sched(dp)->destname, 0);
2296     h[activehd]->used = size - dummy;
2297     h[activehd]->disk->allocated_dumpers--;
2298     adjust_diskspace(dp, DONE);
2299
2300     sched(dp)->dump_attempted += 1;
2301
2302     if((dumper->result != DONE || chunker->result != DONE) &&
2303        sched(dp)->dump_attempted <= 1) {
2304         delete_diskspace(dp);
2305         if (sched(dp)->no_space) {
2306             enqueue_disk(&directq, dp);
2307         } else {
2308             enqueue_disk(&runq, dp);
2309         }
2310     }
2311     else if(size > (off_t)DISK_BLOCK_KB) {
2312         enqueue_disk(&tapeq, dp);
2313     }
2314     else {
2315         delete_diskspace(dp);
2316     }
2317
2318     dumper->busy = 0;
2319     dp->host->inprogress -= 1;
2320     dp->inprogress = 0;
2321
2322     waitpid(chunker->pid, NULL, 0 );
2323     aclose(chunker->fd);
2324     chunker->fd = -1;
2325     chunker->down = 1;
2326
2327     dp = NULL;
2328     if (chunker->result == ABORT_FINISHED)
2329         pending_aborts--;
2330     continue_port_dumps();
2331     /*
2332      * Wakeup any dumpers that are sleeping because of network
2333      * or disk constraints.
2334      */
2335     start_some_dumps(&runq);
2336     startaflush();
2337 }
2338
2339
2340 static void
2341 handle_dumper_result(
2342         void * cookie)
2343 {
2344     /* uses global pending_aborts */
2345     dumper_t *dumper = cookie;
2346     taper_t  *taper;
2347     disk_t *dp, *sdp, *dp1;
2348     cmd_t cmd;
2349     int result_argc;
2350     char *qname;
2351     char **result_argv;
2352
2353     assert(dumper != NULL);
2354     dp = dumper->dp;
2355     assert(dp != NULL);
2356     assert(sched(dp) != NULL);
2357     do {
2358
2359         short_dump_state();
2360
2361         cmd = getresult(dumper->fd, 1, &result_argc, &result_argv);
2362
2363         if(cmd != BOGUS) {
2364             /* result_argv[1] always contains the serial number */
2365             sdp = serial2disk(result_argv[1]);
2366             if (sdp != dp) {
2367                 error(_("Invalid serial number %s"), result_argv[1]);
2368                 g_assert_not_reached();
2369             }
2370         }
2371
2372         qname = quote_string(dp->name);
2373         switch(cmd) {
2374
2375         case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
2376             if(result_argc != 6) {
2377                 error(_("error [dumper DONE result_argc != 6: %d]"), result_argc);
2378                 /*NOTREACHED*/
2379             }
2380
2381             sched(dp)->origsize = OFF_T_ATOI(result_argv[2]);
2382             sched(dp)->dumptime = TIME_T_ATOI(result_argv[4]);
2383
2384             g_printf(_("driver: finished-cmd time %s %s dumped %s:%s\n"),
2385                    walltime_str(curclock()), dumper->name,
2386                    dp->host->hostname, qname);
2387             fflush(stdout);
2388
2389             dumper->result = cmd;
2390
2391             break;
2392
2393         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2394             /*
2395              * Requeue this disk, and fall through to the FAILED
2396              * case for cleanup.
2397              */
2398             if(sched(dp)->dump_attempted) {
2399                 char *qname = quote_string(dp->name);
2400                 char *qerr = quote_string(result_argv[2]);
2401                 log_add(L_FAIL, _("%s %s %s %d [too many dumper retry: %s]"),
2402                     dp->host->hostname, qname, sched(dp)->datestamp,
2403                     sched(dp)->level, qerr);
2404                 g_printf(_("driver: dump failed %s %s %s, too many dumper retry: %s\n"),
2405                         result_argv[1], dp->host->hostname, qname, qerr);
2406                 amfree(qname);
2407                 amfree(qerr);
2408             }
2409             /* FALLTHROUGH */
2410         case FAILED: /* FAILED <handle> <errstr> */
2411             /*free_serial(result_argv[1]);*/
2412             dumper->result = cmd;
2413             break;
2414
2415         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2416             /*
2417              * We sent an ABORT from the NO-ROOM case because this dump
2418              * wasn't going to fit onto the holding disk.  We now need to
2419              * clean up the remains of this image, and try to finish
2420              * other dumps that are waiting on disk space.
2421              */
2422             assert(pending_aborts);
2423             /*free_serial(result_argv[1]);*/
2424             dumper->result = cmd;
2425             break;
2426
2427         case BOGUS:
2428             /* either EOF or garbage from dumper.  Turn it off */
2429             log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2430                     dumper->name, (long)dumper->pid);
2431             if (dumper->ev_read) {
2432                 event_release(dumper->ev_read);
2433                 dumper->ev_read = NULL;
2434             }
2435             aclose(dumper->fd);
2436             dumper->busy = 0;
2437             dumper->down = 1;   /* mark it down so it isn't used again */
2438
2439             /* if it was dumping something, zap it and try again */
2440             if(sched(dp)->dump_attempted) {
2441                 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2442                         dp->host->hostname, qname, sched(dp)->datestamp,
2443                         sched(dp)->level, dumper->name);
2444             } else {
2445                 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2446                         dumper->name, dp->host->hostname, qname,
2447                         sched(dp)->level);
2448             }
2449             dumper->result = cmd;
2450             break;
2451
2452         default:
2453             assert(0);
2454         }
2455         amfree(qname);
2456         g_strfreev(result_argv);
2457
2458         if (cmd != BOGUS) {
2459             int last_dump = 1;
2460             dumper_t *dumper;
2461
2462             run_server_dle_scripts(EXECUTE_ON_POST_DLE_BACKUP,
2463                                get_config_name(), dp, sched(dp)->level);
2464             /* check dump not yet started */
2465             for (dp1=runq.head; dp1 != NULL; dp1 = dp1->next) {
2466                 if (dp1->host == dp->host)
2467                     last_dump = 0;
2468             }
2469             /* check direct to tape dump */
2470             for (dp1=directq.head; dp1 != NULL; dp1 = dp1->next) {
2471                 if (dp1->host == dp->host)
2472                     last_dump = 0;
2473             }
2474             /* check dumping dle */
2475             for (dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
2476                 if (dumper->busy && dumper->dp != dp &&
2477                     dumper->dp->host == dp->host)
2478                  last_dump = 0;
2479             }
2480             if (last_dump && dp->host->post_script == 0) {
2481                 if (dp->host->post_script == 0) {
2482                     run_server_host_scripts(EXECUTE_ON_POST_HOST_BACKUP,
2483                                             get_config_name(), dp->host);
2484                     dp->host->post_script = 1;
2485                 }
2486             }
2487         }
2488
2489         taper = sched(dp)->taper;
2490         /* send the dumper result to the chunker */
2491         if (dumper->chunker) {
2492             if (dumper->chunker->down == 0 && dumper->chunker->fd != -1 &&
2493                 dumper->chunker->result == LAST_TOK) {
2494                 if (cmd == DONE) {
2495                     chunker_cmd(dumper->chunker, DONE, dp, NULL);
2496                 }
2497                 else {
2498                     chunker_cmd(dumper->chunker, FAILED, dp, NULL);
2499                 }
2500             }
2501             if( dumper->result != LAST_TOK &&
2502                 dumper->chunker->result != LAST_TOK)
2503                 dumper_chunker_result(dp);
2504         } else { /* send the dumper result to the taper */
2505             if (cmd == DONE) {
2506                 taper_cmd(DONE, dp, NULL, 0, NULL);
2507             } else {
2508                 taper_cmd(FAILED, dp, NULL, 0, NULL);
2509             }
2510             taper->sendresult = 0;
2511             if (taper->dumper && taper->result != LAST_TOK) {
2512                 dumper_taper_result(dp);
2513             }
2514         }
2515     } while(areads_dataready(dumper->fd));
2516 }
2517
2518
2519 static void
2520 handle_chunker_result(
2521     void *      cookie)
2522 {
2523     chunker_t *chunker = cookie;
2524     assignedhd_t **h=NULL;
2525     dumper_t *dumper;
2526     disk_t *dp, *sdp;
2527     cmd_t cmd;
2528     int result_argc;
2529     char **result_argv;
2530     int dummy;
2531     int activehd = -1;
2532     char *qname;
2533
2534     assert(chunker != NULL);
2535     dumper = chunker->dumper;
2536     assert(dumper != NULL);
2537     dp = dumper->dp;
2538     assert(dp != NULL);
2539     assert(sched(dp) != NULL);
2540     assert(sched(dp)->destname != NULL);
2541     assert(dp != NULL && sched(dp) != NULL && sched(dp)->destname);
2542
2543     if(sched(dp)->holdp) {
2544         h = sched(dp)->holdp;
2545         activehd = sched(dp)->activehd;
2546     }
2547
2548     do {
2549         short_dump_state();
2550
2551         cmd = getresult(chunker->fd, 1, &result_argc, &result_argv);
2552
2553         if(cmd != BOGUS) {
2554             /* result_argv[1] always contains the serial number */
2555             sdp = serial2disk(result_argv[1]);
2556             if (sdp != dp) {
2557                 error(_("Invalid serial number %s"), result_argv[1]);
2558                 g_assert_not_reached();
2559             }
2560         }
2561
2562         switch(cmd) {
2563
2564         case PARTIAL: /* PARTIAL <handle> <dumpsize> <errstr> */
2565         case DONE: /* DONE <handle> <dumpsize> <errstr> */
2566             if(result_argc != 4) {
2567                 error(_("error [chunker %s result_argc != 4: %d]"), cmdstr[cmd],
2568                       result_argc);
2569                 /*NOTREACHED*/
2570             }
2571             /*free_serial(result_argv[1]);*/
2572
2573             sched(dp)->dumpsize = (off_t)atof(result_argv[2]);
2574
2575             qname = quote_string(dp->name);
2576             g_printf(_("driver: finished-cmd time %s %s chunked %s:%s\n"),
2577                    walltime_str(curclock()), chunker->name,
2578                    dp->host->hostname, qname);
2579             fflush(stdout);
2580             amfree(qname);
2581
2582             event_release(chunker->ev_read);
2583
2584             chunker->result = cmd;
2585
2586             break;
2587
2588         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2589             event_release(chunker->ev_read);
2590
2591             chunker->result = cmd;
2592
2593             break;
2594         case FAILED: /* FAILED <handle> <errstr> */
2595             /*free_serial(result_argv[1]);*/
2596
2597             event_release(chunker->ev_read);
2598
2599             chunker->result = cmd;
2600
2601             break;
2602
2603         case NO_ROOM: /* NO-ROOM <handle> <missing_size> */
2604             if (!h || activehd < 0) { /* should never happen */
2605                 error(_("!h || activehd < 0"));
2606                 /*NOTREACHED*/
2607             }
2608             h[activehd]->used -= OFF_T_ATOI(result_argv[2]);
2609             h[activehd]->reserved -= OFF_T_ATOI(result_argv[2]);
2610             h[activehd]->disk->allocated_space -= OFF_T_ATOI(result_argv[2]);
2611             h[activehd]->disk->disksize -= OFF_T_ATOI(result_argv[2]);
2612             break;
2613
2614         case RQ_MORE_DISK: /* RQ-MORE-DISK <handle> */
2615             if (!h || activehd < 0) { /* should never happen */
2616                 error(_("!h || activehd < 0"));
2617                 /*NOTREACHED*/
2618             }
2619             h[activehd]->disk->allocated_dumpers--;
2620             h[activehd]->used = h[activehd]->reserved;
2621             if( h[++activehd] ) { /* There's still some allocated space left.
2622                                    * Tell the dumper about it. */
2623                 sched(dp)->activehd++;
2624                 chunker_cmd( chunker, CONTINUE, dp, NULL );
2625             } else { /* !h[++activehd] - must allocate more space */
2626                 sched(dp)->act_size = sched(dp)->est_size; /* not quite true */
2627                 sched(dp)->est_size = (sched(dp)->act_size/(off_t)20) * (off_t)21; /* +5% */
2628                 sched(dp)->est_size = am_round(sched(dp)->est_size, (off_t)DISK_BLOCK_KB);
2629                 if (sched(dp)->est_size < sched(dp)->act_size + 2*DISK_BLOCK_KB)
2630                     sched(dp)->est_size += 2 * DISK_BLOCK_KB;
2631                 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
2632                                     &dummy,
2633                                     h[activehd-1] );
2634                 if( !h ) {
2635                     /* No diskspace available. The reason for this will be
2636                      * determined in continue_port_dumps(). */
2637                     enqueue_disk( &roomq, dp );
2638                     continue_port_dumps();
2639                     /* continue flush waiting for new tape */
2640                     startaflush();
2641                 } else {
2642                     /* OK, allocate space for disk and have chunker continue */
2643                     sched(dp)->activehd = assign_holdingdisk( h, dp );
2644                     chunker_cmd( chunker, CONTINUE, dp, NULL );
2645                     amfree(h);
2646                 }
2647             }
2648             break;
2649
2650         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
2651             /*
2652              * We sent an ABORT from the NO-ROOM case because this dump
2653              * wasn't going to fit onto the holding disk.  We now need to
2654              * clean up the remains of this image, and try to finish
2655              * other dumps that are waiting on disk space.
2656              */
2657             /*assert(pending_aborts);*/
2658
2659             /*free_serial(result_argv[1]);*/
2660
2661             event_release(chunker->ev_read);
2662
2663             chunker->result = cmd;
2664
2665             break;
2666
2667         case BOGUS:
2668             /* either EOF or garbage from chunker.  Turn it off */
2669             log_add(L_WARNING, _("%s pid %ld is messed up, ignoring it.\n"),
2670                     chunker->name, (long)chunker->pid);
2671
2672             /* if it was dumping something, zap it and try again */
2673             g_assert(h && activehd >= 0);
2674             qname = quote_string(dp->name);
2675             if(sched(dp)->dump_attempted) {
2676                 log_add(L_FAIL, _("%s %s %s %d [%s died]"),
2677                         dp->host->hostname, qname, sched(dp)->datestamp,
2678                         sched(dp)->level, chunker->name);
2679             } else {
2680                 log_add(L_WARNING, _("%s died while dumping %s:%s lev %d."),
2681                         chunker->name, dp->host->hostname, qname,
2682                         sched(dp)->level);
2683             }
2684             amfree(qname);
2685             dp = NULL;
2686
2687             event_release(chunker->ev_read);
2688
2689             chunker->result = cmd;
2690
2691             break;
2692
2693         default:
2694             assert(0);
2695         }
2696         g_strfreev(result_argv);
2697
2698         if(chunker->result != LAST_TOK && chunker->dumper->result != LAST_TOK)
2699             dumper_chunker_result(dp);
2700
2701     } while(areads_dataready(chunker->fd));
2702 }
2703
2704
2705 static void
2706 read_flush(
2707     void *      cookie)
2708 {
2709     sched_t *sp;
2710     disk_t *dp;
2711     int line;
2712     char *hostname, *diskname, *datestamp;
2713     int level;
2714     char *destname;
2715     disk_t *dp1;
2716     char *inpline = NULL;
2717     char *command;
2718     char *s;
2719     int ch;
2720     char *qname = NULL;
2721     char *qdestname = NULL;
2722     char *conf_infofile;
2723
2724     (void)cookie;       /* Quiet unused parameter warning */
2725
2726     event_release(flush_ev_read);
2727     flush_ev_read = NULL;
2728
2729     /* read schedule from stdin */
2730     conf_infofile = config_dir_relative(getconf_str(CNF_INFOFILE));
2731     if (open_infofile(conf_infofile)) {
2732         error(_("could not open info db \"%s\""), conf_infofile);
2733         /*NOTREACHED*/
2734     }
2735     amfree(conf_infofile);
2736
2737     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2738         dumpfile_t file;
2739
2740         line++;
2741         if (inpline[0] == '\0')
2742             continue;
2743
2744         s = inpline;
2745         ch = *s++;
2746
2747         skip_whitespace(s, ch);                 /* find the command */
2748         if(ch == '\0') {
2749             error(_("flush line %d: syntax error (no command)"), line);
2750             /*NOTREACHED*/
2751         }
2752         command = s - 1;
2753         skip_non_whitespace(s, ch);
2754         s[-1] = '\0';
2755
2756         if(strcmp(command,"ENDFLUSH") == 0) {
2757             break;
2758         }
2759
2760         if(strcmp(command,"FLUSH") != 0) {
2761             error(_("flush line %d: syntax error (%s != FLUSH)"), line, command);
2762             /*NOTREACHED*/
2763         }
2764
2765         skip_whitespace(s, ch);                 /* find the hostname */
2766         if(ch == '\0') {
2767             error(_("flush line %d: syntax error (no hostname)"), line);
2768             /*NOTREACHED*/
2769         }
2770         hostname = s - 1;
2771         skip_non_whitespace(s, ch);
2772         s[-1] = '\0';
2773
2774         skip_whitespace(s, ch);                 /* find the diskname */
2775         if(ch == '\0') {
2776             error(_("flush line %d: syntax error (no diskname)"), line);
2777             /*NOTREACHED*/
2778         }
2779         qname = s - 1;
2780         skip_quoted_string(s, ch);
2781         s[-1] = '\0';                           /* terminate the disk name */
2782         diskname = unquote_string(qname);
2783
2784         skip_whitespace(s, ch);                 /* find the datestamp */
2785         if(ch == '\0') {
2786             error(_("flush line %d: syntax error (no datestamp)"), line);
2787             /*NOTREACHED*/
2788         }
2789         datestamp = s - 1;
2790         skip_non_whitespace(s, ch);
2791         s[-1] = '\0';
2792
2793         skip_whitespace(s, ch);                 /* find the level number */
2794         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2795             error(_("flush line %d: syntax error (bad level)"), line);
2796             /*NOTREACHED*/
2797         }
2798         skip_integer(s, ch);
2799
2800         skip_whitespace(s, ch);                 /* find the filename */
2801         if(ch == '\0') {
2802             error(_("flush line %d: syntax error (no filename)"), line);
2803             /*NOTREACHED*/
2804         }
2805         qdestname = s - 1;
2806         skip_quoted_string(s, ch);
2807         s[-1] = '\0';
2808         destname = unquote_string(qdestname);
2809
2810         holding_file_get_dumpfile(destname, &file);
2811         if( file.type != F_DUMPFILE) {
2812             if( file.type != F_CONT_DUMPFILE )
2813                 log_add(L_INFO, _("%s: ignoring cruft file."), destname);
2814             amfree(diskname);
2815             amfree(destname);
2816             dumpfile_free_data(&file);
2817             continue;
2818         }
2819
2820         if(strcmp(hostname, file.name) != 0 ||
2821            strcmp(diskname, file.disk) != 0 ||
2822            strcmp(datestamp, file.datestamp) != 0) {
2823             log_add(L_INFO, _("disk %s:%s not consistent with file %s"),
2824                     hostname, diskname, destname);
2825             amfree(diskname);
2826             amfree(destname);
2827             dumpfile_free_data(&file);
2828             continue;
2829         }
2830         amfree(diskname);
2831
2832         dp = lookup_disk(file.name, file.disk);
2833
2834         if (dp == NULL) {
2835             log_add(L_INFO, _("%s: disk %s:%s not in database, skipping it."),
2836                     destname, file.name, file.disk);
2837             amfree(destname);
2838             dumpfile_free_data(&file);
2839             continue;
2840         }
2841
2842         if (file.dumplevel < 0 || file.dumplevel > 399) {
2843             log_add(L_INFO, _("%s: ignoring file with bogus dump level %d."),
2844                     destname, file.dumplevel);
2845             amfree(destname);
2846             dumpfile_free_data(&file);
2847             continue;
2848         }
2849
2850         if (holding_file_size(destname,1) <= 0) {
2851             log_add(L_INFO, "%s: removing file with no data.", destname);
2852             holding_file_unlink(destname);
2853             amfree(destname);
2854             dumpfile_free_data(&file);
2855             continue;
2856         }
2857
2858         dp1 = (disk_t *)alloc(SIZEOF(disk_t));
2859         *dp1 = *dp;
2860         dp1->next = dp1->prev = NULL;
2861
2862         /* add it to the flushhost list */
2863         if(!flushhost) {
2864             flushhost = alloc(SIZEOF(am_host_t));
2865             flushhost->next = NULL;
2866             flushhost->hostname = stralloc("FLUSHHOST");
2867             flushhost->up = NULL;
2868             flushhost->features = NULL;
2869         }
2870         dp1->hostnext = flushhost->disks;
2871         flushhost->disks = dp1;
2872
2873         sp = (sched_t *) alloc(SIZEOF(sched_t));
2874         sp->destname = destname;
2875         sp->level = file.dumplevel;
2876         sp->dumpdate = NULL;
2877         sp->degr_dumpdate = NULL;
2878         sp->degr_mesg = NULL;
2879         sp->datestamp = stralloc(file.datestamp);
2880         sp->est_nsize = (off_t)0;
2881         sp->est_csize = (off_t)0;
2882         sp->est_time = 0;
2883         sp->est_kps = 10;
2884         sp->origsize = file.orig_size;
2885         sp->priority = 0;
2886         sp->degr_level = -1;
2887         sp->dump_attempted = 0;
2888         sp->taper_attempted = 0;
2889         sp->act_size = holding_file_size(destname, 0);
2890         sp->holdp = build_diskspace(destname);
2891         if(sp->holdp == NULL) continue;
2892         sp->dumper = NULL;
2893         sp->taper = NULL;
2894         sp->timestamp = (time_t)0;
2895
2896         dp1->up = (char *)sp;
2897
2898         enqueue_disk(&tapeq, dp1);
2899         dumpfile_free_data(&file);
2900     }
2901     amfree(inpline);
2902     close_infofile();
2903
2904     startaflush();
2905     if (!nodump) {
2906         schedule_ev_read = event_register((event_id_t)0, EV_READFD,
2907                                           read_schedule, NULL);
2908     } else {
2909         force_flush = 1;
2910     }
2911 }
2912
2913 static void
2914 read_schedule(
2915     void *      cookie)
2916 {
2917     sched_t *sp;
2918     disk_t *dp;
2919     int level, line, priority;
2920     char *dumpdate, *degr_dumpdate, *degr_mesg;
2921     int degr_level;
2922     time_t time, degr_time;
2923     time_t *time_p = &time;
2924     time_t *degr_time_p = &degr_time;
2925     off_t nsize, csize, degr_nsize, degr_csize;
2926     unsigned long kps, degr_kps;
2927     char *hostname, *features, *diskname, *datestamp, *inpline = NULL;
2928     char *command;
2929     char *s;
2930     int ch;
2931     off_t flush_size = (off_t)0;
2932     char *qname = NULL;
2933     long long time_;
2934     long long nsize_;
2935     long long csize_;
2936     long long degr_nsize_;
2937     long long degr_csize_;
2938     GPtrArray *errarray;
2939
2940     (void)cookie;       /* Quiet unused parameter warning */
2941
2942     event_release(schedule_ev_read);
2943     schedule_ev_read = NULL;
2944
2945     /* read schedule from stdin */
2946
2947     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
2948         if (inpline[0] == '\0')
2949             continue;
2950         line++;
2951
2952         s = inpline;
2953         ch = *s++;
2954
2955         skip_whitespace(s, ch);                 /* find the command */
2956         if(ch == '\0') {
2957             error(_("schedule line %d: syntax error (no command)"), line);
2958             /*NOTREACHED*/
2959         }
2960         command = s - 1;
2961         skip_non_whitespace(s, ch);
2962         s[-1] = '\0';
2963
2964         if(strcmp(command,"DUMP") != 0) {
2965             error(_("schedule line %d: syntax error (%s != DUMP)"), line, command);
2966             /*NOTREACHED*/
2967         }
2968
2969         skip_whitespace(s, ch);                 /* find the host name */
2970         if(ch == '\0') {
2971             error(_("schedule line %d: syntax error (no host name)"), line);
2972             /*NOTREACHED*/
2973         }
2974         hostname = s - 1;
2975         skip_non_whitespace(s, ch);
2976         s[-1] = '\0';
2977
2978         skip_whitespace(s, ch);                 /* find the feature list */
2979         if(ch == '\0') {
2980             error(_("schedule line %d: syntax error (no feature list)"), line);
2981             /*NOTREACHED*/
2982         }
2983         features = s - 1;
2984         skip_non_whitespace(s, ch);
2985         s[-1] = '\0';
2986
2987         skip_whitespace(s, ch);                 /* find the disk name */
2988         if(ch == '\0') {
2989             error(_("schedule line %d: syntax error (no disk name)"), line);
2990             /*NOTREACHED*/
2991         }
2992         qname = s - 1;
2993         skip_quoted_string(s, ch);
2994         s[-1] = '\0';                           /* terminate the disk name */
2995         diskname = unquote_string(qname);
2996
2997         skip_whitespace(s, ch);                 /* find the datestamp */
2998         if(ch == '\0') {
2999             error(_("schedule line %d: syntax error (no datestamp)"), line);
3000             /*NOTREACHED*/
3001         }
3002         datestamp = s - 1;
3003         skip_non_whitespace(s, ch);
3004         s[-1] = '\0';
3005
3006         skip_whitespace(s, ch);                 /* find the priority number */
3007         if(ch == '\0' || sscanf(s - 1, "%d", &priority) != 1) {
3008             error(_("schedule line %d: syntax error (bad priority)"), line);
3009             /*NOTREACHED*/
3010         }
3011         skip_integer(s, ch);
3012
3013         skip_whitespace(s, ch);                 /* find the level number */
3014         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
3015             error(_("schedule line %d: syntax error (bad level)"), line);
3016             /*NOTREACHED*/
3017         }
3018         skip_integer(s, ch);
3019
3020         skip_whitespace(s, ch);                 /* find the dump date */
3021         if(ch == '\0') {
3022             error(_("schedule line %d: syntax error (bad dump date)"), line);
3023             /*NOTREACHED*/
3024         }
3025         dumpdate = s - 1;
3026         skip_non_whitespace(s, ch);
3027         s[-1] = '\0';
3028
3029         skip_whitespace(s, ch);                 /* find the native size */
3030         nsize_ = (off_t)0;
3031         if(ch == '\0' || sscanf(s - 1, "%lld", &nsize_) != 1) {
3032             error(_("schedule line %d: syntax error (bad nsize)"), line);
3033             /*NOTREACHED*/
3034         }
3035         nsize = (off_t)nsize_;
3036         skip_integer(s, ch);
3037
3038         skip_whitespace(s, ch);                 /* find the compressed size */
3039         csize_ = (off_t)0;
3040         if(ch == '\0' || sscanf(s - 1, "%lld", &csize_) != 1) {
3041             error(_("schedule line %d: syntax error (bad csize)"), line);
3042             /*NOTREACHED*/
3043         }
3044         csize = (off_t)csize_;
3045         skip_integer(s, ch);
3046
3047         skip_whitespace(s, ch);                 /* find the time number */
3048         if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
3049             error(_("schedule line %d: syntax error (bad estimated time)"), line);
3050             /*NOTREACHED*/
3051         }
3052         *time_p = (time_t)time_;
3053         skip_integer(s, ch);
3054
3055         skip_whitespace(s, ch);                 /* find the kps number */
3056         if(ch == '\0' || sscanf(s - 1, "%lu", &kps) != 1) {
3057             error(_("schedule line %d: syntax error (bad kps)"), line);
3058             continue;
3059         }
3060         skip_integer(s, ch);
3061
3062         degr_dumpdate = NULL;                   /* flag if degr fields found */
3063         skip_whitespace(s, ch);                 /* find the degr level number */
3064         degr_mesg = NULL;
3065         if (ch == '"') {
3066             qname = s - 1;
3067             skip_quoted_string(s, ch);
3068             s[-1] = '\0';                       /* terminate degr mesg */
3069             degr_mesg = unquote_string(qname);
3070             degr_level = -1;
3071             degr_nsize = (off_t)0;
3072             degr_csize = (off_t)0;
3073             degr_time = (time_t)0;
3074             degr_kps = 0;
3075         } else if (ch != '\0') {
3076             if(sscanf(s - 1, "%d", &degr_level) != 1) {
3077                 error(_("schedule line %d: syntax error (bad degr level)"), line);
3078                 /*NOTREACHED*/
3079             }
3080             skip_integer(s, ch);
3081
3082             skip_whitespace(s, ch);             /* find the degr dump date */
3083             if(ch == '\0') {
3084                 error(_("schedule line %d: syntax error (bad degr dump date)"), line);
3085                 /*NOTREACHED*/
3086             }
3087             degr_dumpdate = s - 1;
3088             skip_non_whitespace(s, ch);
3089             s[-1] = '\0';
3090
3091             skip_whitespace(s, ch);             /* find the degr native size */
3092             degr_nsize_ = (off_t)0;
3093             if(ch == '\0'  || sscanf(s - 1, "%lld", &degr_nsize_) != 1) {
3094                 error(_("schedule line %d: syntax error (bad degr nsize)"), line);
3095                 /*NOTREACHED*/
3096             }
3097             degr_nsize = (off_t)degr_nsize_;
3098             skip_integer(s, ch);
3099
3100             skip_whitespace(s, ch);             /* find the degr compressed size */
3101             degr_csize_ = (off_t)0;
3102             if(ch == '\0'  || sscanf(s - 1, "%lld", &degr_csize_) != 1) {
3103                 error(_("schedule line %d: syntax error (bad degr csize)"), line);
3104                 /*NOTREACHED*/
3105             }
3106             degr_csize = (off_t)degr_csize_;
3107             skip_integer(s, ch);
3108
3109             skip_whitespace(s, ch);             /* find the degr time number */
3110             if(ch == '\0' || sscanf(s - 1, "%lld", &time_) != 1) {
3111                 error(_("schedule line %d: syntax error (bad degr estimated time)"), line);
3112                 /*NOTREACHED*/
3113             }
3114             *degr_time_p = (time_t)time_;
3115             skip_integer(s, ch);
3116
3117             skip_whitespace(s, ch);             /* find the degr kps number */
3118             if(ch == '\0' || sscanf(s - 1, "%lu", &degr_kps) != 1) {
3119                 error(_("schedule line %d: syntax error (bad degr kps)"), line);
3120                 /*NOTREACHED*/
3121             }
3122             skip_integer(s, ch);
3123         } else {
3124             error(_("schedule line %d: no degraded estimate or message"), line);
3125         }
3126
3127         dp = lookup_disk(hostname, diskname);
3128         if(dp == NULL) {
3129             log_add(L_WARNING,
3130                     _("schedule line %d: %s:'%s' not in disklist, ignored"),
3131                     line, hostname, qname);
3132             amfree(diskname);
3133             continue;
3134         }
3135
3136         sp = (sched_t *) alloc(SIZEOF(sched_t));
3137         /*@ignore@*/
3138         sp->level = level;
3139         sp->dumpdate = stralloc(dumpdate);
3140         sp->est_nsize = DISK_BLOCK_KB + nsize; /* include header */
3141         sp->est_csize = DISK_BLOCK_KB + csize; /* include header */
3142         /* round estimate to next multiple of DISK_BLOCK_KB */
3143         sp->est_csize = am_round(sp->est_csize, DISK_BLOCK_KB);
3144         sp->est_size = sp->est_csize;
3145         sp->est_time = time;
3146         sp->est_kps = kps;
3147         sp->priority = priority;
3148         sp->datestamp = stralloc(datestamp);
3149
3150         if(degr_dumpdate) {
3151             sp->degr_level = degr_level;
3152             sp->degr_dumpdate = stralloc(degr_dumpdate);
3153             sp->degr_nsize = DISK_BLOCK_KB + degr_nsize;
3154             sp->degr_csize = DISK_BLOCK_KB + degr_csize;
3155             /* round estimate to next multiple of DISK_BLOCK_KB */
3156             sp->degr_csize = am_round(sp->degr_csize, DISK_BLOCK_KB);
3157             sp->degr_time = degr_time;
3158             sp->degr_kps = degr_kps;
3159             sp->degr_mesg = NULL;
3160         } else {
3161             sp->degr_level = -1;
3162             sp->degr_dumpdate = NULL;
3163             sp->degr_mesg = degr_mesg;
3164         }
3165         /*@end@*/
3166
3167         sp->dump_attempted = 0;
3168         sp->taper_attempted = 0;
3169         sp->act_size = 0;
3170         sp->holdp = NULL;
3171         sp->activehd = -1;
3172         sp->dumper = NULL;
3173         sp->taper = NULL;
3174         sp->timestamp = (time_t)0;
3175         sp->destname = NULL;
3176         sp->no_space = 0;
3177
3178         dp->up = (char *) sp;
3179         if(dp->host->features == NULL) {
3180             dp->host->features = am_string_to_feature(features);
3181             if (!dp->host->features) {
3182                 log_add(L_WARNING,
3183                     _("Invalid feature string from client '%s'"),
3184                     features);
3185                 dp->host->features = am_set_default_feature_set();
3186             }
3187         }
3188         remove_disk(&waitq, dp);
3189
3190         errarray = validate_optionstr(dp);
3191         if (errarray->len > 0) {
3192             guint i;
3193             for (i=0; i < errarray->len; i++) {
3194                 log_add(L_FAIL, _("%s %s %s 0 [%s]"),
3195                         dp->host->hostname, qname,
3196                         sp->datestamp,
3197                         (char *)g_ptr_array_index(errarray, i));
3198             }
3199             amfree(qname);
3200         } else {
3201
3202             if (dp->data_path == DATA_PATH_DIRECTTCP &&
3203                 dp->to_holdingdisk == HOLD_AUTO) {
3204                 dp->to_holdingdisk = HOLD_NEVER;
3205             }
3206
3207             if (dp->to_holdingdisk == HOLD_NEVER) {
3208                 enqueue_disk(&directq, dp);
3209             } else {
3210                 enqueue_disk(&runq, dp);
3211             }
3212             flush_size += sp->act_size;
3213         }
3214         amfree(diskname);
3215     }
3216     g_printf(_("driver: flush size %lld\n"), (long long)flush_size);
3217     amfree(inpline);
3218     if(line == 0)
3219         log_add(L_WARNING, _("WARNING: got empty schedule from planner"));
3220     schedule_done = 1;
3221     start_degraded_mode(&runq);
3222     run_server_global_scripts(EXECUTE_ON_PRE_BACKUP, get_config_name());
3223     if (empty(runq)) force_flush = 1;
3224     start_some_dumps(&runq);
3225     startaflush();
3226 }
3227
3228 static unsigned long
3229 free_kps(
3230     netif_t *ip)
3231 {
3232     unsigned long res;
3233
3234     if (ip == NULL) {
3235         netif_t *p;
3236         unsigned long maxusage=0;
3237         unsigned long curusage=0;
3238         for(p = disklist_netifs(); p != NULL; p = p->next) {
3239             maxusage += interface_get_maxusage(p->config);
3240             curusage += p->curusage;
3241         }
3242         if (maxusage >= curusage)
3243             res = maxusage - curusage;
3244         else
3245             res = 0;
3246 #ifndef __lint
3247     } else {
3248         if ((unsigned long)interface_get_maxusage(ip->config) >= ip->curusage)
3249             res = interface_get_maxusage(ip->config) - ip->curusage;
3250         else
3251             res = 0;
3252 #endif
3253     }
3254
3255     return res;
3256 }
3257
3258 static void
3259 interface_state(
3260     char *time_str)
3261 {
3262     netif_t *ip;
3263
3264     g_printf(_("driver: interface-state time %s"), time_str);
3265
3266     for(ip = disklist_netifs(); ip != NULL; ip = ip->next) {
3267         g_printf(_(" if %s: free %lu"), interface_name(ip->config), free_kps(ip));
3268     }
3269     g_printf("\n");
3270 }
3271
3272 static void
3273 allocate_bandwidth(
3274     netif_t *           ip,
3275     unsigned long       kps)
3276 {
3277     ip->curusage += kps;
3278 }
3279
3280 static void
3281 deallocate_bandwidth(
3282     netif_t *           ip,
3283     unsigned long       kps)
3284 {
3285     assert(kps <= ip->curusage);
3286     ip->curusage -= kps;
3287 }
3288
3289 /* ------------ */
3290 static off_t
3291 free_space(void)
3292 {
3293     holdalloc_t *ha;
3294     off_t total_free;
3295     off_t diff;
3296
3297     total_free = (off_t)0;
3298     for(ha = holdalloc; ha != NULL; ha = ha->next) {
3299         diff = ha->disksize - ha->allocated_space;
3300         if(diff > (off_t)0)
3301             total_free += diff;
3302     }
3303     return total_free;
3304 }
3305
3306 /*
3307  * We return an array of pointers to assignedhd_t. The array contains at
3308  * most one entry per holding disk. The list of pointers is terminated by
3309  * a NULL pointer. Each entry contains a pointer to a holdingdisk and
3310  * how much diskspace to use on that disk. Later on, assign_holdingdisk
3311  * will allocate the given amount of space.
3312  * If there is not enough room on the holdingdisks, NULL is returned.
3313  */
3314
3315 static assignedhd_t **
3316 find_diskspace(
3317     off_t               size,
3318     int *               cur_idle,
3319     assignedhd_t *      pref)
3320 {
3321     assignedhd_t **result = NULL;
3322     holdalloc_t *ha, *minp;
3323     int i=0;
3324     int j, minj;
3325     char *used;
3326     off_t halloc, dalloc, hfree, dfree;
3327
3328     (void)cur_idle;     /* Quiet unused parameter warning */
3329
3330     if (size < 2*DISK_BLOCK_KB)
3331         size = 2*DISK_BLOCK_KB;
3332     size = am_round(size, (off_t)DISK_BLOCK_KB);
3333
3334     hold_debug(1, _("find_diskspace: want %lld K\n"),
3335                    (long long)size);
3336
3337     used = alloc(SIZEOF(*used) * num_holdalloc);/*disks used during this run*/
3338     memset( used, 0, (size_t)num_holdalloc );
3339     result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3340     result[0] = NULL;
3341
3342     while( i < num_holdalloc && size > (off_t)0 ) {
3343         /* find the holdingdisk with the fewest active dumpers and among
3344          * those the one with the biggest free space
3345          */
3346         minp = NULL; minj = -1;
3347         for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3348             if( pref && pref->disk == ha && !used[j] &&
3349                 ha->allocated_space <= ha->disksize - (off_t)DISK_BLOCK_KB) {
3350                 minp = ha;
3351                 minj = j;
3352                 break;
3353             }
3354             else if( ha->allocated_space <= ha->disksize - (off_t)(2*DISK_BLOCK_KB) &&
3355                 !used[j] &&
3356                 (!minp ||
3357                  ha->allocated_dumpers < minp->allocated_dumpers ||
3358                  (ha->allocated_dumpers == minp->allocated_dumpers &&
3359                   ha->disksize-ha->allocated_space > minp->disksize-minp->allocated_space)) ) {
3360                 minp = ha;
3361                 minj = j;
3362             }
3363         }
3364
3365         pref = NULL;
3366         if( !minp ) { break; } /* all holding disks are full */
3367         used[minj] = 1;
3368
3369         /* hfree = free space on the disk */
3370         hfree = minp->disksize - minp->allocated_space;
3371
3372         /* dfree = free space for data, remove 1 header for each chunksize */
3373         dfree = hfree - (((hfree-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3374
3375         /* dalloc = space I can allocate for data */
3376         dalloc = ( dfree < size ) ? dfree : size;
3377
3378         /* halloc = space to allocate, including 1 header for each chunksize */
3379         halloc = dalloc + (((dalloc-(off_t)1)/holdingdisk_get_chunksize(minp->hdisk))+(off_t)1) * (off_t)DISK_BLOCK_KB;
3380
3381         hold_debug(1, _("find_diskspace: find diskspace: size %lld hf %lld df %lld da %lld ha %lld\n"),
3382                        (long long)size,
3383                        (long long)hfree,
3384                        (long long)dfree,
3385                        (long long)dalloc,
3386                        (long long)halloc);
3387         size -= dalloc;
3388         result[i] = alloc(SIZEOF(assignedhd_t));
3389         result[i]->disk = minp;
3390         result[i]->reserved = halloc;
3391         result[i]->used = (off_t)0;
3392         result[i]->destname = NULL;
3393         result[i+1] = NULL;
3394         i++;
3395     }
3396     amfree(used);
3397
3398     if(size != (off_t)0) { /* not enough space available */
3399         g_printf(_("find diskspace: not enough diskspace. Left with %lld K\n"), (long long)size);
3400         fflush(stdout);
3401         free_assignedhd(result);
3402         result = NULL;
3403     }
3404
3405     if (debug_holding > 1) {
3406         for( i = 0; result && result[i]; i++ ) {
3407             hold_debug(1, _("find_diskspace: find diskspace: selected %s free %lld reserved %lld dumpers %d\n"),
3408                            holdingdisk_get_diskdir(result[i]->disk->hdisk),
3409                            (long long)(result[i]->disk->disksize -
3410                              result[i]->disk->allocated_space),
3411                            (long long)result[i]->reserved,
3412                            result[i]->disk->allocated_dumpers);
3413         }
3414     }
3415
3416     return result;
3417 }
3418
3419 static int
3420 assign_holdingdisk(
3421     assignedhd_t **     holdp,
3422     disk_t *            diskp)
3423 {
3424     int i, j, c, l=0;
3425     off_t size;
3426     char *sfn = sanitise_filename(diskp->name);
3427     char lvl[64];
3428     assignedhd_t **new_holdp;
3429     char *qname;
3430
3431     g_snprintf( lvl, SIZEOF(lvl), "%d", sched(diskp)->level );
3432
3433     size = am_round(sched(diskp)->est_size - sched(diskp)->act_size,
3434                     (off_t)DISK_BLOCK_KB);
3435
3436     for( c = 0; holdp[c]; c++ )
3437         (void)c; /* count number of disks */
3438
3439     /* allocate memory for sched(diskp)->holdp */
3440     for(j = 0; sched(diskp)->holdp && sched(diskp)->holdp[j]; j++)
3441         (void)j;        /* Quiet lint */
3442     new_holdp = (assignedhd_t **)alloc(SIZEOF(assignedhd_t*)*(j+c+1));
3443     if (sched(diskp)->holdp) {
3444         memcpy(new_holdp, sched(diskp)->holdp, j * SIZEOF(*new_holdp));
3445         amfree(sched(diskp)->holdp);
3446     }
3447     sched(diskp)->holdp = new_holdp;
3448     new_holdp = NULL;
3449
3450     i = 0;
3451     if( j > 0 ) { /* This is a request for additional diskspace. See if we can
3452                    * merge assignedhd_t's */
3453         l=j;
3454         if( sched(diskp)->holdp[j-1]->disk == holdp[0]->disk ) { /* Yes! */
3455             sched(diskp)->holdp[j-1]->reserved += holdp[0]->reserved;
3456             holdp[0]->disk->allocated_space += holdp[0]->reserved;
3457             size = (holdp[0]->reserved>size) ? (off_t)0 : size-holdp[0]->reserved;
3458             qname = quote_string(diskp->name);
3459             hold_debug(1, _("assign_holdingdisk: merging holding disk %s to disk %s:%s, add %lld for reserved %lld, left %lld\n"),
3460                            holdingdisk_get_diskdir(
3461                                                sched(diskp)->holdp[j-1]->disk->hdisk),
3462                            diskp->host->hostname, qname,
3463                            (long long)holdp[0]->reserved,
3464                            (long long)sched(diskp)->holdp[j-1]->reserved,
3465                            (long long)size);
3466             i++;
3467             amfree(qname);
3468             amfree(holdp[0]);
3469             l=j-1;
3470         }
3471     }
3472
3473     /* copy assignedhd_s to sched(diskp), adjust allocated_space */
3474     for( ; holdp[i]; i++ ) {
3475         holdp[i]->destname = newvstralloc( holdp[i]->destname,
3476                                            holdingdisk_get_diskdir(holdp[i]->disk->hdisk), "/",
3477                                            hd_driver_timestamp, "/",
3478                                            diskp->host->hostname, ".",
3479                                            sfn, ".",
3480                                            lvl, NULL );
3481         sched(diskp)->holdp[j++] = holdp[i];
3482         holdp[i]->disk->allocated_space += holdp[i]->reserved;
3483         size = (holdp[i]->reserved > size) ? (off_t)0 :
3484                   (size - holdp[i]->reserved);
3485         qname = quote_string(diskp->name);
3486         hold_debug(1,
3487                    _("assign_holdingdisk: %d assigning holding disk %s to disk %s:%s, reserved %lld, left %lld\n"),
3488                     i, holdingdisk_get_diskdir(holdp[i]->disk->hdisk),
3489                     diskp->host->hostname, qname,
3490                     (long long)holdp[i]->reserved,
3491                     (long long)size);
3492         amfree(qname);
3493         holdp[i] = NULL; /* so it doesn't get free()d... */
3494     }
3495     sched(diskp)->holdp[j] = NULL;
3496     amfree(sfn);
3497
3498     return l;
3499 }
3500
3501 static void
3502 adjust_diskspace(
3503     disk_t *    diskp,
3504     cmd_t       cmd)
3505 {
3506     assignedhd_t **holdp;
3507     off_t total = (off_t)0;
3508     off_t diff;
3509     int i;
3510     char *qname, *hqname, *qdest;
3511
3512     (void)cmd;  /* Quiet unused parameter warning */
3513
3514     qname = quote_string(diskp->name);
3515     qdest = quote_string(sched(diskp)->destname);
3516     hold_debug(1, _("adjust_diskspace: %s:%s %s\n"),
3517                    diskp->host->hostname, qname, qdest);
3518
3519     holdp = sched(diskp)->holdp;
3520
3521     assert(holdp != NULL);
3522
3523     for( i = 0; holdp[i]; i++ ) { /* for each allocated disk */
3524         diff = holdp[i]->used - holdp[i]->reserved;
3525         total += holdp[i]->used;
3526         holdp[i]->disk->allocated_space += diff;
3527         hqname = quote_string(holdingdisk_name(holdp[i]->disk->hdisk));
3528         hold_debug(1, _("adjust_diskspace: hdisk %s done, reserved %lld used %lld diff %lld alloc %lld dumpers %d\n"),
3529                        holdingdisk_name(holdp[i]->disk->hdisk),
3530                        (long long)holdp[i]->reserved,
3531                        (long long)holdp[i]->used,
3532                        (long long)diff,
3533                        (long long)holdp[i]->disk->allocated_space,
3534                        holdp[i]->disk->allocated_dumpers );
3535         holdp[i]->reserved += diff;
3536         amfree(hqname);
3537     }
3538
3539     sched(diskp)->act_size = total;
3540
3541     hold_debug(1, _("adjust_diskspace: after: disk %s:%s used %lld\n"),
3542                    diskp->host->hostname, qname,
3543                    (long long)sched(diskp)->act_size);
3544     amfree(qdest);
3545     amfree(qname);
3546 }
3547
3548 static void
3549 delete_diskspace(
3550     disk_t *diskp)
3551 {
3552     assignedhd_t **holdp;
3553     int i;
3554
3555     holdp = sched(diskp)->holdp;
3556
3557     assert(holdp != NULL);
3558
3559     for( i = 0; holdp[i]; i++ ) { /* for each disk */
3560         /* find all files of this dump on that disk, and subtract their
3561          * reserved sizes from the disk's allocated space
3562          */
3563         holdp[i]->disk->allocated_space -= holdp[i]->used;
3564     }
3565
3566     holding_file_unlink(holdp[0]->destname);    /* no need for the entire list,
3567                                                  * because holding_file_unlink
3568                                                  * will walk through all files
3569                                                  * using cont_filename */
3570     free_assignedhd(sched(diskp)->holdp);
3571     sched(diskp)->holdp = NULL;
3572     sched(diskp)->act_size = (off_t)0;
3573 }
3574
3575 static assignedhd_t **
3576 build_diskspace(
3577     char *      destname)
3578 {
3579     int i, j;
3580     int fd;
3581     size_t buflen;
3582     char buffer[DISK_BLOCK_BYTES];
3583     dumpfile_t file;
3584     assignedhd_t **result;
3585     holdalloc_t *ha;
3586     off_t *used;
3587     char dirname[1000], *ch;
3588     struct stat finfo;
3589     char *filename = destname;
3590
3591     memset(buffer, 0, sizeof(buffer));
3592     used = alloc(SIZEOF(off_t) * num_holdalloc);
3593     for(i=0;i<num_holdalloc;i++)
3594         used[i] = (off_t)0;
3595     result = alloc(SIZEOF(assignedhd_t *) * (num_holdalloc + 1));
3596     result[0] = NULL;
3597     while(filename != NULL && filename[0] != '\0') {
3598         strncpy(dirname, filename, 999);
3599         dirname[999]='\0';
3600         ch = strrchr(dirname,'/');
3601         if (ch) {
3602             *ch = '\0';
3603             ch = strrchr(dirname,'/');
3604             if (ch) {
3605                 *ch = '\0';
3606             }
3607         }
3608
3609         if (!ch) {
3610             g_fprintf(stderr,_("build_diskspace: bogus filename '%s'\n"), filename);
3611             amfree(used);
3612             amfree(result);
3613             return NULL;
3614         }
3615
3616         for(j = 0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3617             if(strcmp(dirname, holdingdisk_get_diskdir(ha->hdisk))==0) {
3618                 break;
3619             }
3620         }
3621         if (!ha || j >= num_holdalloc) {
3622             fprintf(stderr,_("build_diskspace: holding disk file '%s' is not in a holding disk directory.\n"), filename);
3623             amfree(used);
3624             amfree(result);
3625             return NULL;
3626         }
3627         if(stat(filename, &finfo) == -1) {
3628             g_fprintf(stderr, _("build_diskspace: can't stat %s: %s\n"),
3629                       filename, strerror(errno));
3630             amfree(used);
3631             amfree(result);
3632             return NULL;
3633         }
3634         used[j] += ((off_t)finfo.st_size+(off_t)1023)/(off_t)1024;
3635         if((fd = open(filename,O_RDONLY)) == -1) {
3636             g_fprintf(stderr,_("build_diskspace: open of %s failed: %s\n"),
3637                     filename, strerror(errno));
3638             amfree(used);
3639             amfree(result);
3640             return NULL;
3641         }
3642         if ((buflen = full_read(fd, buffer, SIZEOF(buffer))) > 0) {;
3643                 parse_file_header(buffer, &file, buflen);
3644         }
3645         close(fd);
3646         filename = file.cont_filename;
3647     }
3648
3649     for(j = 0, i=0, ha = holdalloc; ha != NULL; ha = ha->next, j++ ) {
3650         if(used[j] != (off_t)0) {
3651             result[i] = alloc(SIZEOF(assignedhd_t));
3652             result[i]->disk = ha;
3653             result[i]->reserved = used[j];
3654             result[i]->used = used[j];
3655             result[i]->destname = stralloc(destname);
3656             result[i+1] = NULL;
3657             i++;
3658         }
3659     }
3660
3661     amfree(used);
3662     return result;
3663 }
3664
3665 static void
3666 holdingdisk_state(
3667     char *      time_str)
3668 {
3669     holdalloc_t *ha;
3670     int dsk;
3671     off_t diff;
3672
3673     g_printf(_("driver: hdisk-state time %s"), time_str);
3674
3675     for(ha = holdalloc, dsk = 0; ha != NULL; ha = ha->next, dsk++) {
3676         diff = ha->disksize - ha->allocated_space;
3677         g_printf(_(" hdisk %d: free %lld dumpers %d"), dsk,
3678                (long long)diff, ha->allocated_dumpers);
3679     }
3680     g_printf("\n");
3681 }
3682
3683 static void
3684 update_failed_dump(
3685     disk_t *    dp)
3686 {
3687     time_t save_timestamp = sched(dp)->timestamp;
3688     /* setting timestamp to 0 removes the current level from the
3689      * database, so that we ensure that it will not be bumped to the
3690      * next level on the next run.  If we didn't do this, dumpdates or
3691      * gnutar-lists might have been updated already, and a bumped
3692      * incremental might be created.  */
3693     sched(dp)->timestamp = 0;
3694     update_info_dumper(dp, (off_t)-1, (off_t)-1, (time_t)-1);
3695     sched(dp)->timestamp = save_timestamp;
3696 }
3697
3698 /* ------------------- */
3699
3700 static int
3701 queue_length(
3702     disklist_t  q)
3703 {
3704     disk_t *p;
3705     int len;
3706
3707     for(len = 0, p = q.head; p != NULL; len++, p = p->next)
3708         (void)len;      /* Quiet lint */
3709     return len;
3710 }
3711
3712 static void
3713 short_dump_state(void)
3714 {
3715     int i, nidle;
3716     char *wall_time;
3717
3718     wall_time = walltime_str(curclock());
3719
3720     g_printf(_("driver: state time %s "), wall_time);
3721     g_printf(_("free kps: %lu space: %lld taper: "),
3722            free_kps(NULL),
3723            (long long)free_space());
3724     if(degraded_mode) g_printf(_("DOWN"));
3725     else {
3726         taper_t *taper;
3727         int writing = 0;
3728         for(taper = tapetable; taper < tapetable+conf_taper_parallel_write;
3729                                taper++) {
3730             if (taper->state & TAPER_STATE_DUMP_TO_TAPE ||
3731                 taper->state & TAPER_STATE_FILE_TO_TAPE)
3732                 writing = 1;
3733         }
3734         if(writing)
3735             g_printf(_("writing"));
3736         else
3737             g_printf(_("idle"));
3738     }
3739     nidle = 0;
3740     for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
3741     g_printf(_(" idle-dumpers: %d"), nidle);
3742     g_printf(_(" qlen tapeq: %d"), queue_length(tapeq));
3743     g_printf(_(" runq: %d"), queue_length(runq));
3744     g_printf(_(" roomq: %d"), queue_length(roomq));
3745     g_printf(_(" wakeup: %d"), (int)sleep_time);
3746     g_printf(_(" driver-idle: %s\n"), _(idle_strings[idle_reason]));
3747     interface_state(wall_time);
3748     holdingdisk_state(wall_time);
3749     fflush(stdout);
3750 }
3751
3752 static TapeAction
3753 tape_action(
3754     taper_t  *taper,
3755     char    **why_no_new_tape)
3756 {
3757     TapeAction result = TAPE_ACTION_NO_ACTION;
3758     dumper_t *dumper;
3759     taper_t  *taper1;
3760     disk_t   *dp;
3761     off_t dumpers_size;
3762     off_t runq_size;
3763     off_t directq_size;
3764     off_t tapeq_size;
3765     off_t sched_size;
3766     off_t dump_to_disk_size;
3767     int   dump_to_disk_terminated;
3768     int   nb_taper_active = nb_sent_new_tape;
3769     int   nb_taper_flushing = 0;
3770     int   dle_free = 0;         /* number of dle that fit on started tape */
3771     int   new_dle = 0;          /* number of dle that doesn't fit on started tape */
3772     off_t new_data = 0;         /* size of dle that doesn't fit on started tape */
3773     off_t data_next_tape = 0;
3774     off_t data_free = 0;
3775     off_t data_lost = 0;
3776     off_t data_lost_next_tape = 0;
3777     gboolean taperflush_criteria;
3778     gboolean flush_criteria;
3779
3780     driver_debug(2, "tape_action: ENTER %p\n", taper);
3781     dumpers_size = 0;
3782     for(dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
3783         if (dumper->busy && !sched(dumper->dp)->taper)
3784             dumpers_size += sched(dumper->dp)->est_size;
3785     }
3786     driver_debug(2, _("dumpers_size: %lld\n"), (long long)dumpers_size);
3787
3788     runq_size = 0;
3789     for(dp = runq.head; dp != NULL; dp = dp->next) {
3790         runq_size += sched(dp)->est_size;
3791     }
3792     driver_debug(2, _("runq_size: %lld\n"), (long long)runq_size);
3793
3794     directq_size = 0;
3795     for(dp = directq.head; dp != NULL; dp = dp->next) {
3796         directq_size += sched(dp)->est_size;
3797     }
3798     driver_debug(2, _("directq_size: %lld\n"), (long long)directq_size);
3799
3800     tapeq_size = directq_size;
3801     for(dp = tapeq.head; dp != NULL; dp = dp->next) {
3802         tapeq_size += sched(dp)->act_size;
3803     }
3804
3805     for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3806          taper1++) {
3807         if (taper1->state & TAPER_STATE_FILE_TO_TAPE ||
3808             taper1->state & TAPER_STATE_DUMP_TO_TAPE) {
3809             nb_taper_flushing++;
3810         }
3811     }
3812
3813     /* Add what is currently written to tape and in the go. */
3814     new_data = 0;
3815     for (taper1 = tapetable; taper1 < tapetable+conf_taper_parallel_write;
3816          taper1++) {
3817         if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3818             if (taper1->nb_dle < conf_max_dle_by_volume) {
3819                 tapeq_size -= taper1->left;
3820             }
3821             dle_free += (conf_max_dle_by_volume - taper1->nb_dle);
3822         }
3823         if (taper1->disk) {
3824             off_t data_to_go;
3825             off_t t_size;
3826             if (taper1->dumper) {
3827                 t_size = sched(taper1->disk)->est_size;
3828             } else {
3829                 t_size = sched(taper1->disk)->act_size;
3830             }
3831             data_to_go =  t_size - taper1->written;
3832             if (data_to_go > taper1->left) {
3833                 data_next_tape += data_to_go - taper1->left;
3834                 data_lost += taper1->written + taper1->left;
3835                 if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3836                     dle_free -= (conf_max_dle_by_volume - taper1->nb_dle) + 1;
3837                 } else {
3838                     dle_free -= 2;
3839                     new_data += t_size;
3840                 }
3841             } else {
3842                 if (!(taper1->state & TAPER_STATE_TAPE_STARTED)) {
3843                     dle_free--;
3844                     new_data += t_size;
3845                 }
3846                 data_free += taper1->left - data_to_go;
3847             }
3848             tapeq_size += data_to_go;
3849         }
3850     }
3851
3852     new_dle = queue_length(tapeq) - dle_free;
3853     driver_debug(2, _("dle_free: %d\n"), dle_free);
3854     driver_debug(2, _("new_dle: %d\n"), new_dle);
3855     if (new_dle > 0) {
3856         if (taperflush == 0 &&
3857             flush_threshold_dumped == 0 &&
3858             flush_threshold_scheduled == 0) {
3859             /* shortcut, will trigger taperflush_criteria and/or flush_criteria */
3860             new_data += 1;
3861         } else {
3862             /* sum the size of the first new-dle in tapeq */
3863             /* they should be the reverse taperalgo       */
3864             for (dp = tapeq.head;
3865                  dp != NULL && new_dle > 0;
3866                  dp = dp->next, new_dle--) {
3867                 new_data += sched(dp)->act_size;
3868             }
3869         }
3870         if (tapeq_size < new_data) {
3871             tapeq_size = new_data;
3872         }
3873     }
3874     driver_debug(2, _("new_data: %lld\n"), (long long)new_data);
3875     data_lost_next_tape = tape_length + data_free - data_next_tape - runq_size - directq_size - tapeq_size;
3876     driver_debug(2, _("data_lost: %lld\n"), (long long)data_lost);
3877     driver_debug(2, _("data_free: %lld\n"), (long long)data_free);
3878     driver_debug(2, _("data_next_tape: %lld\n"), (long long)data_next_tape);
3879     driver_debug(2, _("data_lost_next_tape: %lld\n"), (long long)data_lost_next_tape);
3880 ;
3881     driver_debug(2, _("tapeq_size: %lld\n"), (long long)tapeq_size);
3882
3883     sched_size = runq_size + directq_size + tapeq_size + dumpers_size;
3884     driver_debug(2, _("sched_size: %lld\n"), (long long)sched_size);
3885
3886     dump_to_disk_size = dumpers_size + runq_size + directq_size;
3887     driver_debug(2, _("dump_to_disk_size: %lld\n"), (long long)dump_to_disk_size);
3888
3889     dump_to_disk_terminated = schedule_done && dump_to_disk_size == 0;
3890
3891     for (taper1 = tapetable; taper1 < tapetable + conf_taper_parallel_write;
3892                              taper1++) {
3893         if (taper1->state & TAPER_STATE_TAPE_STARTED) {
3894             nb_taper_active++;
3895         }
3896     }
3897
3898     taperflush_criteria = (taperflush < tapeq_size &&
3899                            (force_flush == 1 || dump_to_disk_terminated));
3900     flush_criteria = (flush_threshold_dumped < tapeq_size &&
3901                       flush_threshold_scheduled < sched_size) ||
3902                      taperflush_criteria;
3903
3904     driver_debug(2, "taperflush %lld\n", (long long)taperflush);
3905     driver_debug(2, "flush_threshold_dumped %lld\n", (long long)flush_threshold_dumped);
3906     driver_debug(2, "flush_threshold_scheduled %lld\n", (long long)flush_threshold_scheduled);
3907     driver_debug(2, "force_flush %d\n", force_flush);
3908     driver_debug(2, "dump_to_disk_terminated %d\n", dump_to_disk_terminated);
3909     driver_debug(2, "queue_length(runq) %d\n", queue_length(runq));
3910     driver_debug(2, "queue_length(directq) %d\n", queue_length(directq));
3911     driver_debug(2, "queue_length(tapeq) %d\n", queue_length(tapeq));
3912     driver_debug(2, "taperflush_criteria %d\n", taperflush_criteria);
3913     driver_debug(2, "flush_criteria %d\n", flush_criteria);
3914
3915     // Changing conditionals can produce a driver hang, take care.
3916     //
3917     // when to start writting to a new tape
3918     if (taper->state & TAPER_STATE_TAPE_REQUESTED) {
3919         driver_debug(2, "tape_action: TAPER_STATE_TAPE_REQUESTED\n");
3920         if (current_tape >= conf_runtapes && taper_nb_scan_volume == 0 &&
3921             nb_taper_active == 0) {
3922             *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3923                 "does not allow additional tapes"), current_tape, conf_runtapes);
3924             driver_debug(2, "tape_action: TAPER_STATE_TAPE_REQUESTED return TAPE_ACTION_NO_NEW_TAPE\n");
3925             result |= TAPE_ACTION_NO_NEW_TAPE;
3926         } else if (current_tape < conf_runtapes &&
3927                    taper_nb_scan_volume == 0 &&
3928                    (flush_criteria ||
3929                     (data_lost > data_lost_next_tape) ||
3930                     nb_taper_active == 0) &&
3931                    (last_started_taper == NULL ||
3932                     last_started_taper == taper)) {
3933             driver_debug(2, "tape_action: TAPER_STATE_TAPE_REQUESTED return TAPE_ACTION_SCAN\n");
3934             result |= TAPE_ACTION_SCAN;
3935         } else {
3936             driver_debug(2, "tape_action: TAPER_STATE_TAPE_REQUESTED return TAPE_ACTION_MOVE\n");
3937             result |= TAPE_ACTION_MOVE;
3938         }
3939     } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3940         ((taper->state & TAPER_STATE_DUMP_TO_TAPE) ||   // for dump to tape
3941          !empty(directq) ||                             // if a dle is waiting for a dump to tape
3942          !empty(roomq) ||                               // holding disk constraint
3943          idle_reason == IDLE_NO_DISKSPACE ||            // holding disk constraint
3944          flush_criteria ||                              // flush criteria
3945          data_lost > data_lost_next_tape
3946         )) {
3947         driver_debug(2, "tape_action: TAPER_STATE_WAIT_FOR_TAPE return TAPE_ACTION_NEW_TAPE\n");
3948         result |= TAPE_ACTION_NEW_TAPE;
3949     // when to stop using new tape
3950     } else if ((taper->state & TAPER_STATE_WAIT_FOR_TAPE) &&
3951                (taperflush >= tapeq_size &&             // taperflush criteria
3952                (force_flush == 1 ||                     //  if force_flush
3953                 dump_to_disk_terminated))               //  or all dump to disk
3954               ) {
3955         driver_debug(2, "tape_action: TAPER_STATE_WAIT_FOR_TAPE B\n");
3956         if (nb_taper_active <= 0) {
3957             if (current_tape >= conf_runtapes) {
3958                 *why_no_new_tape = g_strdup_printf(_("%d tapes filled; runtapes=%d "
3959                       "does not allow additional tapes"), current_tape, conf_runtapes);
3960                 driver_debug(2, "tape_action: TAPER_STATE_WAIT_FOR_TAPE return TAPE_ACTION_NO_NEW_TAPE\n");
3961                 result |= TAPE_ACTION_NO_NEW_TAPE;
3962             } else if (dumpers_size <= 0) {
3963                 *why_no_new_tape = _("taperflush criteria not met");
3964                 driver_debug(2, "tape_action: TAPER_STATE_WAIT_FOR_TAPE return TAPE_ACTION_NO_NEW_TAPE\n");
3965                 result |= TAPE_ACTION_NO_NEW_TAPE;
3966             }
3967         }
3968     }
3969
3970     // when to start a flush
3971     if (taper->state & TAPER_STATE_IDLE) {
3972         driver_debug(2, "tape_action: TAPER_STATE_IDLE\n");
3973         if (!degraded_mode && (!empty(tapeq) || !empty(directq)) &&
3974             (taper->state & TAPER_STATE_TAPE_STARTED ||         // tape already started
3975              !empty(roomq) ||                                   // holding disk constraint
3976              idle_reason == IDLE_NO_DISKSPACE ||                // holding disk constraint
3977              flush_criteria)) {                                 // flush
3978
3979             if (nb_taper_flushing == 0) {
3980                 driver_debug(2, "tape_action: TAPER_STATE_IDLE return TAPE_ACTION_START_A_FLUSH\n");
3981                 result |= TAPE_ACTION_START_A_FLUSH;
3982             } else {
3983                 driver_debug(2, "tape_action: TAPER_STATE_IDLE return TAPE_ACTION_START_A_FLUSH_FIT\n");
3984                 result |= TAPE_ACTION_START_A_FLUSH_FIT;
3985             }
3986         } else {
3987             driver_debug(2, "tape_action: TAPER_STATE_IDLE return TAPE_ACTION_NO_ACTION\n");
3988         }
3989     }
3990     return result;
3991 }
3992
3993 static int
3994 no_taper_flushing(void)
3995 {
3996     taper_t *taper;
3997
3998     for (taper = tapetable; taper < tapetable + conf_taper_parallel_write;
3999          taper++) {
4000         if (taper->state & TAPER_STATE_FILE_TO_TAPE)
4001             return 0;
4002     }
4003     return 1;
4004 }
4005
4006 static int
4007 active_dumper(void)
4008 {
4009     int i, nidle=0;
4010
4011     for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
4012     return inparallel - nidle;
4013 }
4014 #if 0
4015 static void
4016 dump_state(
4017     const char *str)
4018 {
4019     int i;
4020     disk_t *dp;
4021     char *qname;
4022
4023     g_printf("================\n");
4024     g_printf(_("driver state at time %s: %s\n"), walltime_str(curclock()), str);
4025     g_printf(_("free kps: %lu, space: %lld\n"),
4026            free_kps(NULL),
4027            (long long)free_space());
4028     if(degraded_mode) g_printf(_("taper: DOWN\n"));
4029     else if(taper->status == TAPER_IDLE) g_printf(_("taper: idle\n"));
4030     else g_printf(_("taper: writing %s:%s.%d est size %lld\n"),
4031                 taper->disk->host->hostname, taper->disk->name,
4032                 sched(taper->disk)->level,
4033                 (long long)sched(taper->disk)->est_size);
4034     for(i = 0; i < inparallel; i++) {
4035         dp = dmptable[i].dp;
4036         if(!dmptable[i].busy)
4037           g_printf(_("%s: idle\n"), dmptable[i].name);
4038         else
4039           qname = quote_string(dp->name);
4040           g_printf(_("%s: dumping %s:%s.%d est kps %d size %lld time %lu\n"),
4041                 dmptable[i].name, dp->host->hostname, qname, sched(dp)->level,
4042                 sched(dp)->est_kps, (long long)sched(dp)->est_size, sched(dp)->est_time);
4043           amfree(qname);
4044     }
4045     dump_queue("TAPE", tapeq, 5, stdout);
4046     dump_queue("ROOM", roomq, 5, stdout);
4047     dump_queue("RUN ", runq, 5, stdout);
4048     g_printf("================\n");
4049     fflush(stdout);
4050 }
4051 #endif