git.gag.com Git - debian/amanda/blob - server-src/driver.c

   1 /*
   2  * Amanda, The Advanced Maryland Automatic Network Disk Archiver
   3  * Copyright (c) 1991-1998 University of Maryland at College Park
   4  * All Rights Reserved.
   5  *
   6  * Permission to use, copy, modify, distribute, and sell this software and its
   7  * documentation for any purpose is hereby granted without fee, provided that
   8  * the above copyright notice appear in all copies and that both that
   9  * copyright notice and this permission notice appear in supporting
  10  * documentation, and that the name of U.M. not be used in advertising or
  11  * publicity pertaining to distribution of the software without specific,
  12  * written prior permission.  U.M. makes no representations about the
  13  * suitability of this software for any purpose.  It is provided "as is"
  14  * without express or implied warranty.
  15  *
  16  * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
  18  * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  19  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  20  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  21  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22  *
  23  * Authors: the Amanda Development Team.  Its members are listed in a
  24  * file named AUTHORS, in the root directory of this distribution.
  25  */
  26 /*
  27  * $Id: driver.c,v 1.198.2.6 2006/12/27 14:44:48 martinea Exp $
  28  *
  29  * controlling process for the Amanda backup system
  30  */
  31
  32 /*
  33  * XXX possibly modify tape queue to be cognizant of how much room is left on
  34  *     tape.  Probably not effective though, should do this in planner.
  35  */
  36
  37 #define HOLD_DEBUG
  38
  39 #include "amanda.h"
  40 #include "clock.h"
  41 #include "conffile.h"
  42 #include "diskfile.h"
  43 #include "event.h"
  44 #include "holding.h"
  45 #include "infofile.h"
  46 #include "logfile.h"
  47 #include "statfs.h"
  48 #include "version.h"
  49 #include "driverio.h"
  50 #include "server_util.h"
  51
  52 static disklist_t waitq, runq, tapeq, roomq;
  53 static int pending_aborts;
  54 static disk_t *taper_disk;
  55 static int degraded_mode;
  56 static off_t reserved_space;
  57 static off_t total_disksize;
  58 static char *dumper_program;
  59 static char *chunker_program;
  60 static int  inparallel;
  61 static int nodump = 0;
  62 static off_t tape_length = (off_t)0;
  63 static off_t tape_left = (off_t)0;
  64 static int current_tape = 1;
  65 static int conf_taperalgo;
  66 static int conf_runtapes;
  67 static time_t sleep_time;
  68 static int idle_reason;
  69 static char *driver_timestamp;
  70 static char *hd_driver_timestamp;
  71 static am_host_t *flushhost = NULL;
  72 static int need_degraded=0;
  73
  74 static event_handle_t *dumpers_ev_time = NULL;
  75 static event_handle_t *schedule_ev_read = NULL;
  76
  77 static int wait_children(int count);
  78 static void wait_for_children(void);
  79 static void allocate_bandwidth(interface_t *ip, unsigned long kps);
  80 static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp);
  81 static void adjust_diskspace(disk_t *diskp, cmd_t cmd);
  82 static void delete_diskspace(disk_t *diskp);
  83 static assignedhd_t **build_diskspace(char *destname);
  84 static int client_constrained(disk_t *dp);
  85 static void deallocate_bandwidth(interface_t *ip, unsigned long kps);
  86 static void dump_schedule(disklist_t *qp, char *str);
  87 static int dump_to_tape(disk_t *dp);
  88 static assignedhd_t **find_diskspace(off_t size, int *cur_idle,
  89                                         assignedhd_t *preferred);
  90 static unsigned long free_kps(interface_t *ip);
  91 static off_t free_space(void);
  92 static void dumper_result(disk_t *dp);
  93 static void handle_dumper_result(void *);
  94 static void handle_chunker_result(void *);
  95 static void handle_dumpers_time(void *);
  96 static void handle_taper_result(void *);
  97 static void holdingdisk_state(char *time_str);
  98 static dumper_t *idle_dumper(void);
  99 static void interface_state(char *time_str);
 100 static int queue_length(disklist_t q);
 101 static disklist_t read_flush(void);
 102 static void read_schedule(void *cookie);
 103 static void short_dump_state(void);
 104 static void startaflush(void);
 105 static void start_degraded_mode(disklist_t *queuep);
 106 static void start_some_dumps(disklist_t *rq);
 107 static void continue_port_dumps(void);
 108 static void update_failed_dump_to_tape(disk_t *);
 109 #if 0
 110 static void dump_state(const char *str);
 111 #endif
 112 int main(int main_argc, char **main_argv);
 113
 114 static const char *idle_strings[] = {
 115 #define NOT_IDLE                0
 116     "not-idle",
 117 #define IDLE_NO_DUMPERS         1
 118     "no-dumpers",
 119 #define IDLE_START_WAIT         2
 120     "start-wait",
 121 #define IDLE_NO_HOLD            3
 122     "no-hold",
 123 #define IDLE_CLIENT_CONSTRAINED 4
 124     "client-constrained",
 125 #define IDLE_NO_DISKSPACE       5
 126     "no-diskspace",
 127 #define IDLE_TOO_LARGE          6
 128     "file-too-large",
 129 #define IDLE_NO_BANDWIDTH       7
 130     "no-bandwidth",
 131 #define IDLE_TAPER_WAIT         8
 132     "taper-wait",
 133 };
 134
 135 int
 136 main(
 137     int         main_argc,
 138     char **     main_argv)
 139 {
 140     disklist_t origq;
 141     disk_t *diskp;
 142     int dsk;
 143     dumper_t *dumper;
 144     char *newdir = NULL;
 145     generic_fs_stats_t fs;
 146     holdingdisk_t *hdp;
 147     unsigned long malloc_hist_1, malloc_size_1;
 148     unsigned long malloc_hist_2, malloc_size_2;
 149     unsigned long reserve = 100;
 150     char *conffile;
 151     char *conf_diskfile;
 152     cmd_t cmd;
 153     int result_argc;
 154     char *result_argv[MAX_ARGS+1];
 155     char *taper_program;
 156     char *conf_tapetype;
 157     tapetype_t *tape;
 158     char *line;
 159     int    new_argc,   my_argc;
 160     char **new_argv, **my_argv;
 161
 162     safe_fd(-1, 0);
 163
 164     setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0);
 165     setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0);
 166
 167     set_pname("driver");
 168
 169     dbopen(DBG_SUBDIR_SERVER);
 170
 171     atexit(wait_for_children);
 172
 173     /* Don't die when child closes pipe */
 174     signal(SIGPIPE, SIG_IGN);
 175
 176     malloc_size_1 = malloc_inuse(&malloc_hist_1);
 177
 178     erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE);
 179     set_logerror(logerror);
 180
 181     startclock();
 182
 183     parse_server_conf(main_argc, main_argv, &new_argc, &new_argv);
 184     my_argc = new_argc;
 185     my_argv = new_argv;
 186
 187     printf("%s: pid %ld executable %s version %s\n",
 188            get_pname(), (long) getpid(), my_argv[0], version());
 189
 190     if (my_argc > 1) {
 191         config_name = stralloc(my_argv[1]);
 192         config_dir = vstralloc(CONFIG_DIR, "/", config_name, "/", NULL);
 193         if(my_argc > 2) {
 194             if(strncmp(my_argv[2], "nodump", 6) == 0) {
 195                 nodump = 1;
 196             }
 197         }
 198
 199     } else {
 200
 201         char my_cwd[STR_SIZE];
 202
 203         if (getcwd(my_cwd, SIZEOF(my_cwd)) == NULL) {
 204             error("cannot determine current working directory");
 205             /*NOTREACHED*/
 206         }
 207         config_dir = stralloc2(my_cwd, "/");
 208         if ((config_name = strrchr(my_cwd, '/')) != NULL) {
 209             config_name = stralloc(config_name + 1);
 210         }
 211     }
 212
 213     safe_cd();
 214
 215     conffile = stralloc2(config_dir, CONFFILE_NAME);
 216     if(read_conffile(conffile)) {
 217         error("errors processing config file \"%s\"", conffile);
 218         /*NOTREACHED*/
 219     }
 220     amfree(conffile);
 221
 222     dbrename(config_name, DBG_SUBDIR_SERVER);
 223
 224     report_bad_conf_arg();
 225
 226     amfree(driver_timestamp);
 227     /* read timestamp from stdin */
 228     while ((line = agets(stdin)) != NULL) {
 229         if (line[0] != '\0')
 230             break;
 231         amfree(line);
 232     }
 233     if ( line == NULL ) {
 234       error("Did not get DATE line from planner");
 235       /*NOTREACHED*/
 236     }
 237     driver_timestamp = alloc(15);
 238     strncpy(driver_timestamp, &line[5], 14);
 239     driver_timestamp[14] = '\0';
 240     amfree(line);
 241     log_add(L_START,"date %s", driver_timestamp);
 242
 243     /* check that we don't do many dump in a day and usetimestamps is off */
 244     if(strlen(driver_timestamp) == 8) {
 245         if (!nodump) {
 246             char *conf_logdir = getconf_str(CNF_LOGDIR);
 247             char *logfile    = vstralloc(conf_logdir, "/log.",
 248                                          driver_timestamp, ".0", NULL);
 249             char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.",
 250                                          driver_timestamp, ".0", NULL);
 251             if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) {
 252                 log_add(L_WARNING, "WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day.");
 253             }
 254             amfree(oldlogfile);
 255             amfree(logfile);
 256         }
 257         hd_driver_timestamp = construct_timestamp(NULL);
 258     }
 259     else {
 260         hd_driver_timestamp = stralloc(driver_timestamp);
 261     }
 262
 263     taper_program = vstralloc(libexecdir, "/", "taper", versionsuffix(), NULL);
 264     dumper_program = vstralloc(libexecdir, "/", "dumper", versionsuffix(),
 265                                NULL);
 266     chunker_program = vstralloc(libexecdir, "/", "chunker", versionsuffix(),
 267                                NULL);
 268
 269     conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO);
 270     conf_tapetype = getconf_str(CNF_TAPETYPE);
 271     conf_runtapes = getconf_int(CNF_RUNTAPES);
 272     tape = lookup_tapetype(conf_tapetype);
 273     tape_length = tapetype_get_length(tape);
 274     printf("driver: tape size " OFF_T_FMT "\n", (OFF_T_FMT_TYPE)tape_length);
 275
 276     /* start initializing: read in databases */
 277
 278     conf_diskfile = getconf_str(CNF_DISKFILE);
 279     if (*conf_diskfile == '/') {
 280         conf_diskfile = stralloc(conf_diskfile);
 281     } else {
 282         conf_diskfile = stralloc2(config_dir, conf_diskfile);
 283     }
 284     if (read_diskfile(conf_diskfile, &origq) < 0) {
 285         error("could not load disklist \"%s\"", conf_diskfile);
 286         /*NOTREACHED*/
 287     }
 288     amfree(conf_diskfile);
 289
 290     /* set up any configuration-dependent variables */
 291
 292     inparallel  = getconf_int(CNF_INPARALLEL);
 293
 294     reserve = (unsigned long)getconf_int(CNF_RESERVE);
 295
 296     total_disksize = (off_t)0;
 297     for(hdp = getconf_holdingdisks(), dsk = 0; hdp != NULL; hdp = hdp->next, dsk++) {
 298         hdp->up = (void *)alloc(SIZEOF(holdalloc_t));
 299         holdalloc(hdp)->allocated_dumpers = 0;
 300         holdalloc(hdp)->allocated_space = (off_t)0;
 301
 302         if(get_fs_stats(holdingdisk_get_diskdir(hdp), &fs) == -1
 303            || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) {
 304             log_add(L_WARNING, "WARNING: ignoring holding disk %s: %s\n",
 305                     holdingdisk_get_diskdir(hdp), strerror(errno));
 306             hdp->disksize = 0L;
 307             continue;
 308         }
 309
 310         if(fs.avail != (off_t)-1) {
 311             if(hdp->disksize > (off_t)0) {
 312                 if(hdp->disksize > fs.avail) {
 313                     log_add(L_WARNING,
 314                             "WARNING: %s: " OFF_T_FMT " KB requested, "
 315                             "but only " OFF_T_FMT " KB available.",
 316                             holdingdisk_get_diskdir(hdp),
 317                             (OFF_T_FMT_TYPE)hdp->disksize,
 318                             (OFF_T_FMT_TYPE)fs.avail);
 319                             hdp->disksize = fs.avail;
 320                 }
 321             }
 322             else if((fs.avail + hdp->disksize) < (off_t)0) {
 323                 log_add(L_WARNING,
 324                         "WARNING: %s: not " OFF_T_FMT " KB free.",
 325                         holdingdisk_get_diskdir(hdp), -hdp->disksize);
 326                 hdp->disksize = (off_t)0;
 327                 continue;
 328             }
 329             else
 330                 hdp->disksize += fs.avail;
 331         }
 332
 333         printf("driver: adding holding disk %d dir %s size "
 334                 OFF_T_FMT " chunksize " OFF_T_FMT "\n",
 335                dsk, holdingdisk_get_diskdir(hdp),
 336                (OFF_T_FMT_TYPE)hdp->disksize,
 337                (OFF_T_FMT_TYPE)(holdingdisk_get_chunksize(hdp)));
 338
 339         newdir = newvstralloc(newdir,
 340                               holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp,
 341                               NULL);
 342         if(!mkholdingdir(newdir)) {
 343             hdp->disksize = (off_t)0;
 344         }
 345         total_disksize += hdp->disksize;
 346     }
 347
 348     reserved_space = total_disksize * (off_t)(reserve / 100);
 349
 350     printf("reserving " OFF_T_FMT " out of " OFF_T_FMT
 351            " for degraded-mode dumps\n",
 352            (OFF_T_FMT_TYPE)reserved_space, (OFF_T_FMT_TYPE)free_space());
 353
 354     amfree(newdir);
 355
 356     if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS;
 357
 358     /* taper takes a while to get going, so start it up right away */
 359
 360     init_driverio();
 361     if(conf_runtapes > 0) {
 362         startup_tape_process(taper_program);
 363         taper_cmd(START_TAPER, driver_timestamp, NULL, 0, NULL);
 364     }
 365
 366     /* fire up the dumpers now while we are waiting */
 367     if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp);
 368
 369     /*
 370      * Read schedule from stdin.  Usually, this is a pipe from planner,
 371      * so the effect is that we wait here for the planner to
 372      * finish, but meanwhile the taper is rewinding the tape, reading
 373      * the label, checking it, writing a new label and all that jazz
 374      * in parallel with the planner.
 375      */
 376
 377     runq.head = NULL;
 378     runq.tail = NULL;
 379     waitq = origq;
 380     tapeq = read_flush();
 381
 382     roomq.head = roomq.tail = NULL;
 383
 384     log_add(L_STATS, "startup time %s", walltime_str(curclock()));
 385
 386     printf("driver: start time %s inparallel %d bandwidth %lu diskspace "
 387            OFF_T_FMT " ", walltime_str(curclock()), inparallel,
 388            free_kps((interface_t *)0), (OFF_T_FMT_TYPE)free_space());
 389     printf(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n",
 390            "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo),
 391            getconf_str(CNF_DUMPORDER));
 392     fflush(stdout);
 393
 394     /* ok, planner is done, now lets see if the tape is ready */
 395
 396     if(conf_runtapes > 0) {
 397         cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
 398
 399         if(cmd != TAPER_OK) {
 400             /* no tape, go into degraded mode: dump to holding disk */
 401             need_degraded=1;
 402         }
 403     }
 404     else {
 405         need_degraded=1;
 406     }
 407
 408     tape_left = tape_length;
 409     taper_busy = 0;
 410     taper_disk = NULL;
 411     taper_ev_read = NULL;
 412     if(!need_degraded) startaflush();
 413
 414     if(!nodump)
 415         schedule_ev_read = event_register((event_id_t)0, EV_READFD, read_schedule, NULL);
 416
 417     short_dump_state();
 418     event_loop(0);
 419
 420     /* handle any remaining dumps by dumping directly to tape, if possible */
 421
 422     while(!empty(runq) && taper > 0) {
 423         diskp = dequeue_disk(&runq);
 424         if (diskp->to_holdingdisk == HOLD_REQUIRED) {
 425             log_add(L_FAIL, "%s %s %s %d [%s]",
 426                 diskp->host->hostname, diskp->name, sched(diskp)->datestamp,
 427                 sched(diskp)->level,
 428                 "can't dump required holdingdisk");
 429         }
 430         else if (!degraded_mode) {
 431             int rc = dump_to_tape(diskp);
 432             if(rc == 1)
 433                 log_add(L_INFO,
 434                         "%s %s %d [dump to tape failed, will try again]",
 435                         diskp->host->hostname,
 436                         diskp->name,
 437                         sched(diskp)->level);
 438             else if(rc == 2)
 439                 log_add(L_FAIL, "%s %s %s %d [dump to tape failed]",
 440                         diskp->host->hostname,
 441                         diskp->name,
 442                         sched(diskp)->datestamp,
 443                         sched(diskp)->level);
 444         }
 445         else
 446             log_add(L_FAIL, "%s %s %s %d [%s]",
 447                 diskp->host->hostname, diskp->name, sched(diskp)->datestamp,
 448                 sched(diskp)->level,
 449                 diskp->to_holdingdisk == HOLD_AUTO ?
 450                     "no more holding disk space" :
 451                     "can't dump no-hold disk in degraded mode");
 452     }
 453
 454     short_dump_state();                         /* for amstatus */
 455
 456     printf("driver: QUITTING time %s telling children to quit\n",
 457            walltime_str(curclock()));
 458     fflush(stdout);
 459
 460     if(!nodump) {
 461         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 462             if(dumper->fd >= 0)
 463                 dumper_cmd(dumper, QUIT, NULL);
 464         }
 465     }
 466
 467     if(taper >= 0) {
 468         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 469     }
 470
 471     /* wait for all to die */
 472     wait_children(600);
 473
 474     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
 475         cleanup_holdingdisk(holdingdisk_get_diskdir(hdp), 0);
 476         amfree(hdp->up);
 477     }
 478     amfree(newdir);
 479
 480     check_unfree_serial();
 481     printf("driver: FINISHED time %s\n", walltime_str(curclock()));
 482     fflush(stdout);
 483     log_add(L_FINISH,"date %s time %s", driver_timestamp, walltime_str(curclock()));
 484     amfree(driver_timestamp);
 485
 486     free_new_argv(new_argc, new_argv);
 487     amfree(dumper_program);
 488     amfree(taper_program);
 489     amfree(config_dir);
 490     amfree(config_name);
 491
 492     malloc_size_2 = malloc_inuse(&malloc_hist_2);
 493
 494     if(malloc_size_1 != malloc_size_2) {
 495         malloc_list(fileno(stderr), malloc_hist_1, malloc_hist_2);
 496     }
 497
 498     dbclose();
 499
 500     return 0;
 501 }
 502
 503 /* sleep up to count seconds, and wait for terminating child process */
 504 /* if sleep is negative, this function will not timeout              */
 505 /* exit once all child process are finished or the timout expired    */
 506 /* return 0 if no more children to wait                              */
 507 /* return 1 if some children are still alive                         */
 508 static int
 509 wait_children(int count)
 510 {
 511     pid_t     pid;
 512     amwait_t  retstat;
 513     char     *who;
 514     char     *what;
 515     int       code=0;
 516     dumper_t *dumper;
 517     int       wait_errno;
 518
 519     do {
 520         do {
 521             pid = waitpid((pid_t)-1, &retstat, WNOHANG);
 522             wait_errno = errno;
 523             if (pid > 0) {
 524                 what = NULL;
 525                 if (! WIFEXITED(retstat)) {
 526                     what = "signal";
 527                     code = WTERMSIG(retstat);
 528                 } else if (WEXITSTATUS(retstat) != 0) {
 529                     what = "code";
 530                     code = WEXITSTATUS(retstat);
 531                 }
 532                 who = NULL;
 533                 for (dumper = dmptable; dumper < dmptable + inparallel;
 534                      dumper++) {
 535                     if (pid == dumper->pid) {
 536                         who = stralloc(dumper->name);
 537                         dumper->pid = -1;
 538                         break;
 539                     }
 540                     if (dumper->chunker && pid == dumper->chunker->pid) {
 541                         who = stralloc(dumper->chunker->name);
 542                         dumper->chunker->pid = -1;
 543                         break;
 544                     }
 545                 }
 546                 if (who == NULL && pid == taper_pid) {
 547                     who = stralloc("taper");
 548                     taper_pid = -1;
 549                 }
 550                 if(what != NULL && who == NULL) {
 551                     who = stralloc("unknown");
 552                 }
 553                 if(who && what) {
 554                     log_add(L_WARNING, "%s pid %u exited with %s %d\n", who,
 555                             (unsigned)pid, what, code);
 556                     printf("driver: %s pid %u exited with %s %d\n", who,
 557                            (unsigned)pid, what, code);
 558                 }
 559                 amfree(who);
 560             }
 561         } while (pid > 0 || wait_errno == EINTR);
 562         if (errno != ECHILD)
 563             sleep(1);
 564         if (count > 0)
 565             count--;
 566     } while ((errno != ECHILD) && (count != 0));
 567     return (errno != ECHILD);
 568 }
 569
 570 static void
 571 kill_children(int signal)
 572 {
 573     dumper_t *dumper;
 574
 575     if(!nodump) {
 576         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 577             if (!dumper->down && dumper->pid > 1) {
 578                 printf("driver: sending signal %d to %s pid %u\n", signal,
 579                        dumper->name, (unsigned)dumper->pid);
 580                 if (kill(dumper->pid, signal) == -1 && errno == ESRCH) {
 581                     if (dumper->chunker)
 582                         dumper->chunker->pid = 0;
 583                 }
 584                 if (dumper->chunker && dumper->chunker->pid > 1) {
 585                     printf("driver: sending signal %d to %s pid %u\n", signal,
 586                            dumper->chunker->name,
 587                            (unsigned)dumper->chunker->pid);
 588                     if (kill(dumper->chunker->pid, signal) == -1 &&
 589                         errno == ESRCH)
 590                         dumper->chunker->pid = 0;
 591                 }
 592             }
 593         }
 594     }
 595
 596     if(taper_pid > 1)
 597         printf("driver: sending signal %d to %s pid %u\n", signal,
 598                "taper", (unsigned)taper_pid);
 599         if (kill(taper_pid, signal) == -1 && errno == ESRCH)
 600             taper_pid = 0;
 601 }
 602
 603 static void
 604 wait_for_children(void)
 605 {
 606     dumper_t *dumper;
 607
 608     if(!nodump) {
 609         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 610             if (dumper->pid > 1 && dumper->fd >= 0) {
 611                 dumper_cmd(dumper, QUIT, NULL);
 612                 if (dumper->chunker && dumper->chunker->pid > 1 &&
 613                     dumper->chunker->fd >= 0)
 614                     chunker_cmd(dumper->chunker, QUIT, NULL);
 615             }
 616         }
 617     }
 618
 619     if(taper_pid > 1 && taper > 0) {
 620         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 621     }
 622
 623     if(wait_children(60) == 0)
 624         return;
 625
 626     kill_children(SIGHUP);
 627     if(wait_children(60) == 0)
 628         return;
 629
 630     kill_children(SIGKILL);
 631     if(wait_children(-1) == 0)
 632         return;
 633
 634 }
 635
 636 static void
 637 startaflush(void)
 638 {
 639     disk_t *dp = NULL;
 640     disk_t *fit = NULL;
 641     char *datestamp;
 642     int extra_tapes = 0;
 643     char *qname;
 644
 645     if(!degraded_mode && !taper_busy && !empty(tapeq)) {
 646
 647         datestamp = sched(tapeq.head)->datestamp;
 648         switch(conf_taperalgo) {
 649         case ALGO_FIRST:
 650                 dp = dequeue_disk(&tapeq);
 651                 break;
 652         case ALGO_FIRSTFIT:
 653                 fit = tapeq.head;
 654                 while (fit != NULL) {
 655                     extra_tapes = (fit->tape_splitsize > (off_t)0) ?
 656                                         conf_runtapes - current_tape : 0;
 657                     if(sched(fit)->act_size <= (tape_left +
 658                              tape_length * (off_t)extra_tapes) &&
 659                              strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 660                         dp = fit;
 661                         fit = NULL;
 662                     }
 663                     else {
 664                         fit = fit->next;
 665                     }
 666                 }
 667                 if(dp) remove_disk(&tapeq, dp);
 668                 break;
 669         case ALGO_LARGEST:
 670                 fit = dp = tapeq.head;
 671                 while (fit != NULL) {
 672                     if(sched(fit)->act_size > sched(dp)->act_size &&
 673                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 674                         dp = fit;
 675                     }
 676                     fit = fit->next;
 677                 }
 678                 if(dp) remove_disk(&tapeq, dp);
 679                 break;
 680         case ALGO_LARGESTFIT:
 681                 fit = tapeq.head;
 682                 while (fit != NULL) {
 683                     extra_tapes = (fit->tape_splitsize > (off_t)0) ?
 684                                         conf_runtapes - current_tape : 0;
 685                     if(sched(fit)->act_size <=
 686                        (tape_left + tape_length * (off_t)extra_tapes) &&
 687                        (!dp || sched(fit)->act_size > sched(dp)->act_size) &&
 688                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 689                         dp = fit;
 690                     }
 691                     fit = fit->next;
 692                 }
 693                 if(dp) remove_disk(&tapeq, dp);
 694                 break;
 695         case ALGO_SMALLEST:
 696                 break;
 697         case ALGO_LAST:
 698                 dp = tapeq.tail;
 699                 remove_disk(&tapeq, dp);
 700                 break;
 701         }
 702         if(!dp) { /* ALGO_SMALLEST, or default if nothing fit. */
 703             if(conf_taperalgo != ALGO_SMALLEST)  {
 704                 fprintf(stderr,
 705                    "driver: startaflush: Using SMALLEST because nothing fit\n");
 706             }
 707             fit = dp = tapeq.head;
 708             while (fit != NULL) {
 709                 if(sched(fit)->act_size < sched(dp)->act_size &&
 710                    strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 711                     dp = fit;
 712                 }
 713                 fit = fit->next;
 714             }
 715             if(dp) remove_disk(&tapeq, dp);
 716         }
 717         if(taper_ev_read == NULL) {
 718             taper_ev_read = event_register((event_id_t)taper, EV_READFD,
 719                                            handle_taper_result, NULL);
 720         }
 721         if (dp) {
 722             taper_disk = dp;
 723             taper_busy = 1;
 724             qname = quote_string(dp->name);
 725             taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level,
 726                       sched(dp)->datestamp);
 727             fprintf(stderr,"driver: startaflush: %s %s %s "
 728                     OFF_T_FMT " " OFF_T_FMT "\n",
 729                     taperalgo2str(conf_taperalgo), dp->host->hostname, qname,
 730                     (OFF_T_FMT_TYPE)sched(taper_disk)->act_size,
 731                     (OFF_T_FMT_TYPE)tape_left);
 732             if(sched(dp)->act_size <= tape_left)
 733                 tape_left -= sched(dp)->act_size;
 734             else
 735                 tape_left = (off_t)0;
 736             amfree(qname);
 737         } else {
 738             error("FATAL: Taper marked busy and no work found.");
 739             /*NOTREACHED*/
 740         }
 741     } else if(!taper_busy && taper_ev_read != NULL) {
 742         event_release(taper_ev_read);
 743         taper_ev_read = NULL;
 744     }
 745 }
 746
 747
 748 static int
 749 client_constrained(
 750     disk_t *    dp)
 751 {
 752     disk_t *dp2;
 753
 754     /* first, check if host is too busy */
 755
 756     if(dp->host->inprogress >= dp->host->maxdumps) {
 757         return 1;
 758     }
 759
 760     /* next, check conflict with other dumps on same spindle */
 761
 762     if(dp->spindle == -1) {     /* but spindle -1 never conflicts by def. */
 763         return 0;
 764     }
 765
 766     for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext)
 767         if(dp2->inprogress && dp2->spindle == dp->spindle) {
 768             return 1;
 769         }
 770
 771     return 0;
 772 }
 773
 774 static void
 775 start_some_dumps(
 776     disklist_t *        rq)
 777 {
 778     int cur_idle;
 779     disk_t *diskp, *delayed_diskp, *diskp_accept;
 780     assignedhd_t **holdp=NULL, **holdp_accept;
 781     const time_t now = time(NULL);
 782     cmd_t cmd;
 783     int result_argc;
 784     char *result_argv[MAX_ARGS+1];
 785     chunker_t *chunker;
 786     dumper_t *dumper;
 787     char dumptype;
 788     char *dumporder;
 789
 790     idle_reason = IDLE_NO_DUMPERS;
 791     sleep_time = 0;
 792
 793     if(dumpers_ev_time != NULL) {
 794         event_release(dumpers_ev_time);
 795         dumpers_ev_time = NULL;
 796     }
 797
 798     for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) {
 799
 800         if( dumper->busy ) {
 801             continue;
 802         }
 803
 804         if (dumper->ev_read != NULL) {
 805             event_release(dumper->ev_read);
 806             dumper->ev_read = NULL;
 807         }
 808
 809         /*
 810          * A potential problem with starting from the bottom of the dump time
 811          * distribution is that a slave host will have both one of the shortest
 812          * and one of the longest disks, so starting its shortest disk first will
 813          * tie up the host and eliminate its longest disk from consideration the
 814          * first pass through.  This could cause a big delay in starting that long
 815          * disk, which could drag out the whole night's dumps.
 816          *
 817          * While starting from the top of the dump time distribution solves the
 818          * above problem, this turns out to be a bad idea, because the big dumps
 819          * will almost certainly pack the holding disk completely, leaving no
 820          * room for even one small dump to start.  This ends up shutting out the
 821          * small-end dumpers completely (they stay idle).
 822          *
 823          * The introduction of multiple simultaneous dumps to one host alleviates
 824          * the biggest&smallest dumps problem: both can be started at the
 825          * beginning.
 826          */
 827
 828         diskp_accept = NULL;
 829         holdp_accept = NULL;
 830         delayed_diskp = NULL;
 831
 832         cur_idle = NOT_IDLE;
 833
 834         dumporder = getconf_str(CNF_DUMPORDER);
 835         if(strlen(dumporder) > (size_t)(dumper-dmptable)) {
 836             dumptype = dumporder[dumper-dmptable];
 837         }
 838         else {
 839             if(dumper-dmptable < 3)
 840                 dumptype = 't';
 841             else
 842                 dumptype = 'T';
 843         }
 844
 845         for(diskp = rq->head; diskp != NULL; diskp = diskp->next) {
 846             assert(diskp->host != NULL && sched(diskp) != NULL);
 847
 848             if (diskp->host->start_t > now) {
 849                 cur_idle = max(cur_idle, IDLE_START_WAIT);
 850                 if (delayed_diskp == NULL || sleep_time > diskp->host->start_t) {
 851                     delayed_diskp = diskp;
 852                     sleep_time = diskp->host->start_t;
 853                 }
 854             } else if(diskp->start_t > now) {
 855                 cur_idle = max(cur_idle, IDLE_START_WAIT);
 856                 if (delayed_diskp == NULL || sleep_time > diskp->start_t) {
 857                     delayed_diskp = diskp;
 858                     sleep_time = diskp->start_t;
 859                 }
 860             } else if (diskp->host->netif->curusage > 0 &&
 861                        sched(diskp)->est_kps > free_kps(diskp->host->netif)) {
 862                 cur_idle = max(cur_idle, IDLE_NO_BANDWIDTH);
 863             } else if(sched(diskp)->no_space) {
 864                 cur_idle = max(cur_idle, IDLE_NO_DISKSPACE);
 865             } else if (diskp->to_holdingdisk == HOLD_NEVER) {
 866                 cur_idle = max(cur_idle, IDLE_NO_HOLD);
 867             } else if ((holdp =
 868                 find_diskspace(sched(diskp)->est_size, &cur_idle, NULL)) == NULL) {
 869                 cur_idle = max(cur_idle, IDLE_NO_DISKSPACE);
 870             } else if (client_constrained(diskp)) {
 871                 free_assignedhd(holdp);
 872                 cur_idle = max(cur_idle, IDLE_CLIENT_CONSTRAINED);
 873             } else {
 874
 875                 /* disk fits, dump it */
 876                 int accept = !diskp_accept;
 877                 if(!accept) {
 878                     switch(dumptype) {
 879                       case 's': accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size);
 880                                 break;
 881                       case 'S': accept = (sched(diskp)->est_size > sched(diskp_accept)->est_size);
 882                                 break;
 883                       case 't': accept = (sched(diskp)->est_time < sched(diskp_accept)->est_time);
 884                                 break;
 885                       case 'T': accept = (sched(diskp)->est_time > sched(diskp_accept)->est_time);
 886                                 break;
 887                       case 'b': accept = (sched(diskp)->est_kps < sched(diskp_accept)->est_kps);
 888                                 break;
 889                       case 'B': accept = (sched(diskp)->est_kps > sched(diskp_accept)->est_kps);
 890                                 break;
 891                       default:  log_add(L_WARNING, "Unknown dumporder character \'%c\', using 's'.\n",
 892                                         dumptype);
 893                                 accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size);
 894                                 break;
 895                     }
 896                 }
 897                 if(accept) {
 898                     if( !diskp_accept || !degraded_mode || diskp->priority >= diskp_accept->priority) {
 899                         if(holdp_accept) free_assignedhd(holdp_accept);
 900                         diskp_accept = diskp;
 901                         holdp_accept = holdp;
 902                     }
 903                     else {
 904                         free_assignedhd(holdp);
 905                     }
 906                 }
 907                 else {
 908                     free_assignedhd(holdp);
 909                 }
 910             }
 911         }
 912
 913         diskp = diskp_accept;
 914         holdp = holdp_accept;
 915
 916         idle_reason = max(idle_reason, cur_idle);
 917
 918         /*
 919          * If we have no disk at this point, and there are disks that
 920          * are delayed, then schedule a time event to call this dumper
 921          * with the disk with the shortest delay.
 922          */
 923         if (diskp == NULL && delayed_diskp != NULL) {
 924             assert(sleep_time > now);
 925             sleep_time -= now;
 926             dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME,
 927                 handle_dumpers_time, &runq);
 928             return;
 929         } else if (diskp != NULL) {
 930             sched(diskp)->act_size = (off_t)0;
 931             allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
 932             sched(diskp)->activehd = assign_holdingdisk(holdp, diskp);
 933             amfree(holdp);
 934             sched(diskp)->destname = newstralloc(sched(diskp)->destname,
 935                                                  sched(diskp)->holdp[0]->destname);
 936             diskp->host->inprogress++;  /* host is now busy */
 937             diskp->inprogress = 1;
 938             sched(diskp)->dumper = dumper;
 939             sched(diskp)->timestamp = now;
 940
 941             dumper->busy = 1;           /* dumper is now busy */
 942             dumper->dp = diskp;         /* link disk to dumper */
 943             remove_disk(rq, diskp);             /* take it off the run queue */
 944
 945             sched(diskp)->origsize = (off_t)-1;
 946             sched(diskp)->dumpsize = (off_t)-1;
 947             sched(diskp)->dumptime = (time_t)0;
 948             sched(diskp)->tapetime = (time_t)0;
 949             chunker = dumper->chunker;
 950             chunker->result = LAST_TOK;
 951             dumper->result = LAST_TOK;
 952             startup_chunk_process(chunker,chunker_program);
 953             chunker_cmd(chunker, START, (void *)driver_timestamp);
 954             chunker->dumper = dumper;
 955             chunker_cmd(chunker, PORT_WRITE, diskp);
 956             cmd = getresult(chunker->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
 957             if(cmd != PORT) {
 958                 assignedhd_t **h=NULL;
 959                 int activehd;
 960
 961                 printf("driver: did not get PORT from %s for %s:%s\n",
 962                        chunker->name, diskp->host->hostname, diskp->name);
 963                 fflush(stdout);
 964
 965                 deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
 966                 h = sched(diskp)->holdp;
 967                 activehd = sched(diskp)->activehd;
 968                 h[activehd]->used = 0;
 969                 holdalloc(h[activehd]->disk)->allocated_dumpers--;
 970                 adjust_diskspace(diskp, DONE);
 971                 delete_diskspace(diskp);
 972                 diskp->host->inprogress--;
 973                 diskp->inprogress = 0;
 974                 sched(diskp)->dumper = NULL;
 975                 dumper->busy = 0;
 976                 dumper->dp = NULL;
 977                 sched(diskp)->attempted++;
 978                 free_serial_dp(diskp);
 979                 if(sched(diskp)->attempted < 2)
 980                     enqueue_disk(rq, diskp);
 981             }
 982             else {
 983                 dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD,
 984                                                  handle_dumper_result, dumper);
 985                 chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD,
 986                                                    handle_chunker_result, chunker);
 987                 dumper->output_port = atoi(result_argv[2]);
 988
 989                 dumper_cmd(dumper, PORT_DUMP, diskp);
 990             }
 991             diskp->host->start_t = now + 15;
 992         }
 993     }
 994 }
 995
 996 /*
 997  * This gets called when a dumper is delayed for some reason.  It may
 998  * be because a disk has a delayed start, or amanda is constrained
 999  * by network or disk limits.
1000  */
1001
1002 static void
1003 handle_dumpers_time(
1004     void *      cookie)
1005 {
1006     disklist_t *runq = cookie;
1007     event_release(dumpers_ev_time);
1008     dumpers_ev_time = NULL;
1009     start_some_dumps(runq);
1010 }
1011
1012 static void
1013 dump_schedule(
1014     disklist_t *qp,
1015     char *      str)
1016 {
1017     disk_t *dp;
1018     char *qname;
1019
1020     printf("dump of driver schedule %s:\n--------\n", str);
1021
1022     for(dp = qp->head; dp != NULL; dp = dp->next) {
1023         qname = quote_string(dp->name);
1024         printf("  %-20s %-25s lv %d t %5lu s " OFF_T_FMT " p %d\n",
1025                dp->host->hostname, qname, sched(dp)->level,
1026                sched(dp)->est_time,
1027                (OFF_T_FMT_TYPE)sched(dp)->est_size, sched(dp)->priority);
1028         amfree(qname);
1029     }
1030     printf("--------\n");
1031 }
1032
1033 static void
1034 start_degraded_mode(
1035     /*@keep@*/ disklist_t *queuep)
1036 {
1037     disk_t *dp;
1038     disklist_t newq;
1039     off_t est_full_size;
1040     char *qname;
1041
1042     if (taper_ev_read != NULL) {
1043         event_release(taper_ev_read);
1044         taper_ev_read = NULL;
1045     }
1046
1047     newq.head = newq.tail = 0;
1048
1049     dump_schedule(queuep, "before start degraded mode");
1050
1051     est_full_size = (off_t)0;
1052     while(!empty(*queuep)) {
1053         dp = dequeue_disk(queuep);
1054
1055         qname = quote_string(dp->name);
1056         if(sched(dp)->level != 0)
1057             /* go ahead and do the disk as-is */
1058             enqueue_disk(&newq, dp);
1059         else {
1060             if (reserved_space + est_full_size + sched(dp)->est_size
1061                 <= total_disksize) {
1062                 enqueue_disk(&newq, dp);
1063                 est_full_size += sched(dp)->est_size;
1064             }
1065             else if(sched(dp)->degr_level != -1) {
1066                 sched(dp)->level = sched(dp)->degr_level;
1067                 sched(dp)->dumpdate = sched(dp)->degr_dumpdate;
1068                 sched(dp)->est_nsize = sched(dp)->degr_nsize;
1069                 sched(dp)->est_csize = sched(dp)->degr_csize;
1070                 sched(dp)->est_time = sched(dp)->degr_time;
1071                 sched(dp)->est_kps  = sched(dp)->degr_kps;
1072                 enqueue_disk(&newq, dp);
1073             }
1074             else {
1075                 log_add(L_FAIL,"%s %s %s %d [can't switch to incremental dump]",
1076                         dp->host->hostname, qname, sched(dp)->datestamp,
1077                         sched(dp)->level);
1078             }
1079         }
1080         amfree(qname);
1081     }
1082
1083     /*@i@*/ *queuep = newq;
1084     degraded_mode = 1;
1085
1086     dump_schedule(queuep, "after start degraded mode");
1087 }
1088
1089
1090 static void
1091 continue_port_dumps(void)
1092 {
1093     disk_t *dp, *ndp;
1094     assignedhd_t **h;
1095     int active_dumpers=0, busy_dumpers=0, i;
1096     dumper_t *dumper;
1097
1098     /* First we try to grant diskspace to some dumps waiting for it. */
1099     for( dp = roomq.head; dp; dp = ndp ) {
1100         ndp = dp->next;
1101         /* find last holdingdisk used by this dump */
1102         for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) {
1103             (void)h; /* Quiet lint */
1104         }
1105         /* find more space */
1106         h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1107                             &active_dumpers, h[i] );
1108         if( h ) {
1109             for(dumper = dmptable; dumper < dmptable + inparallel &&
1110                                    dumper->dp != dp; dumper++) {
1111                 (void)dp; /* Quiet lint */
1112             }
1113             assert( dumper < dmptable + inparallel );
1114             sched(dp)->activehd = assign_holdingdisk( h, dp );
1115             chunker_cmd( dumper->chunker, CONTINUE, dp );
1116             amfree(h);
1117             remove_disk( &roomq, dp );
1118         }
1119     }
1120
1121     /* So for some disks there is less holding diskspace available than
1122      * was asked for. Possible reasons are
1123      * a) diskspace has been allocated for other dumps which are
1124      *    still running or already being written to tape
1125      * b) all other dumps have been suspended due to lack of diskspace
1126      * c) this dump doesn't fit on all the holding disks
1127      * Case a) is not a problem. We just wait for the diskspace to
1128      * be freed by moving the current disk to a queue.
1129      * If case b) occurs, we have a deadlock situation. We select
1130      * a dump from the queue to be aborted and abort it. It will
1131      * be retried later dumping to disk.
1132      * If case c) is detected, the dump is aborted. Next time
1133      * it will be dumped directly to tape. Actually, case c is a special
1134      * manifestation of case b) where only one dumper is busy.
1135      */
1136     for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1137         if( dumper->busy ) {
1138             busy_dumpers++;
1139             if( !find_disk(&roomq, dumper->dp) ) {
1140                 active_dumpers++;
1141             } else if( !dp ||
1142                        sched(dp)->est_size > sched(dumper->dp)->est_size ) {
1143                 dp = dumper->dp;
1144             }
1145         }
1146     }
1147     if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) &&
1148         ((!taper_busy && empty(tapeq)) || degraded_mode) &&
1149         pending_aborts == 0 ) { /* not case a */
1150         if( busy_dumpers == 1 ) { /* case c */
1151             sched(dp)->no_space = 1;
1152         }
1153         /* case b */
1154         /* At this time, dp points to the dump with the smallest est_size.
1155          * We abort that dump, hopefully not wasting too much time retrying it.
1156          */
1157         remove_disk( &roomq, dp );
1158         chunker_cmd( sched(dp)->dumper->chunker, ABORT, NULL);
1159         dumper_cmd( sched(dp)->dumper, ABORT, NULL );
1160         pending_aborts++;
1161     }
1162 }
1163
1164
1165 static void
1166 handle_taper_result(
1167     void *      cookie)
1168 {
1169     disk_t *dp;
1170     off_t filenum;
1171     cmd_t cmd;
1172     int result_argc;
1173     char *result_argv[MAX_ARGS+1];
1174     int avail_tapes = 0;
1175
1176     (void)cookie;       /* Quiet unused parameter warning */
1177
1178     assert(cookie == NULL);
1179
1180     do {
1181
1182         short_dump_state();
1183
1184         cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
1185
1186         switch(cmd) {
1187
1188         case PARTIAL:
1189         case DONE:      /* DONE <handle> <label> <tape file> <err mess> */
1190             if(result_argc != 5) {
1191                 error("error: [taper DONE result_argc != 5: %d", result_argc);
1192                 /*NOTREACHED*/
1193             }
1194
1195             dp = serial2disk(result_argv[2]);
1196             free_serial(result_argv[2]);
1197
1198             filenum = OFF_T_ATOI(result_argv[4]);
1199             if(cmd == DONE) {
1200                 update_info_taper(dp, result_argv[3], filenum,
1201                                   sched(dp)->level);
1202             }
1203
1204             delete_diskspace(dp);
1205
1206             printf("driver: finished-cmd time %s taper wrote %s:%s\n",
1207                    walltime_str(curclock()), dp->host->hostname, dp->name);
1208             fflush(stdout);
1209
1210             amfree(sched(dp)->destname);
1211             amfree(sched(dp)->dumpdate);
1212             amfree(sched(dp)->degr_dumpdate);
1213             amfree(sched(dp)->datestamp);
1214             amfree(dp->up);
1215
1216             taper_busy = 0;
1217             taper_disk = NULL;
1218             startaflush();
1219
1220             /* continue with those dumps waiting for diskspace */
1221             continue_port_dumps();
1222             break;
1223
1224         case TRYAGAIN:  /* TRY-AGAIN <handle> <err mess> */
1225             if (result_argc < 2) {
1226                 error("error [taper TRYAGAIN result_argc < 2: %d]",
1227                       result_argc);
1228                 /*NOTREACHED*/
1229             }
1230             dp = serial2disk(result_argv[2]);
1231             free_serial(result_argv[2]);
1232             printf("driver: taper-tryagain time %s disk %s:%s\n",
1233                    walltime_str(curclock()), dp->host->hostname, dp->name);
1234             fflush(stdout);
1235
1236             /* See how many tapes we have left, but we alwyays
1237                retry once (why?) */
1238             current_tape++;
1239             if(dp->tape_splitsize > (off_t)0)
1240                 avail_tapes = conf_runtapes - current_tape;
1241             else
1242                 avail_tapes = 0;
1243
1244             if(sched(dp)->attempted > avail_tapes) {
1245                 log_add(L_FAIL, "%s %s %s %d [too many taper retries]",
1246                         dp->host->hostname, dp->name, sched(dp)->datestamp,
1247                         sched(dp)->level);
1248                 printf("driver: taper failed %s %s %s, too many taper retry\n",
1249                        result_argv[2], dp->host->hostname, dp->name);
1250             }
1251             else {
1252                 /* Re-insert into taper queue. */
1253                 sched(dp)->attempted++;
1254                 headqueue_disk(&tapeq, dp);
1255             }
1256
1257             tape_left = tape_length;
1258
1259             /* run next thing from queue */
1260
1261             taper_busy = 0;
1262             taper_disk = NULL;
1263             startaflush();
1264             continue_port_dumps();
1265             break;
1266
1267         case SPLIT_CONTINUE:  /* SPLIT_CONTINUE <handle> <new_label> */
1268             if (result_argc != 3) {
1269                 error("error [taper SPLIT_CONTINUE result_argc != 3: %d]",
1270                       result_argc);
1271                 /*NOTREACHED*/
1272             }
1273
1274             break;
1275         case SPLIT_NEEDNEXT:  /* SPLIT-NEEDNEXT <handle> <kb written> */
1276             if (result_argc != 3) {
1277                 error("error [taper SPLIT_NEEDNEXT result_argc != 3: %d]",
1278                       result_argc);
1279                 /*NOTREACHED*/
1280             }
1281
1282             /* Update our tape counter and reset tape_left */
1283             current_tape++;
1284             tape_left = tape_length;
1285
1286             /* Reduce the size of the dump by amount written and reduce
1287                tape_left by the amount left over */
1288             dp = serial2disk(result_argv[2]);
1289             sched(dp)->act_size -= OFF_T_ATOI(result_argv[3]);
1290             if (sched(dp)->act_size < tape_left)
1291                 tape_left -= sched(dp)->act_size;
1292             else
1293                 tape_length = 0;
1294
1295             break;
1296
1297         case TAPE_ERROR: /* TAPE-ERROR <handle> <err mess> */
1298             dp = serial2disk(result_argv[2]);
1299             free_serial(result_argv[2]);
1300             printf("driver: finished-cmd time %s taper wrote %s:%s\n",
1301                    walltime_str(curclock()), dp->host->hostname, dp->name);
1302             fflush(stdout);
1303             log_add(L_WARNING, "Taper  error: %s", result_argv[3]);
1304             /*FALLTHROUGH*/
1305
1306         case BOGUS:
1307             if (cmd == BOGUS) {
1308                 log_add(L_WARNING, "Taper protocol error");
1309             }
1310             /*
1311              * Since we received a taper error, we can't send anything more
1312              * to the taper.  Go into degraded mode to try to get everthing
1313              * onto disk.  Later, these dumps can be flushed to a new tape.
1314              * The tape queue is zapped so that it appears empty in future
1315              * checks. If there are dumps waiting for diskspace to be freed,
1316              * cancel one.
1317              */
1318             if(!nodump) {
1319                 log_add(L_WARNING,
1320                         "going into degraded mode because of taper component error.");
1321                 start_degraded_mode(&runq);
1322             }
1323             tapeq.head = tapeq.tail = NULL;
1324             taper_busy = 0;
1325             taper_disk = NULL;
1326             if(taper_ev_read != NULL) {
1327                 event_release(taper_ev_read);
1328                 taper_ev_read = NULL;
1329             }
1330             if(cmd != TAPE_ERROR) aclose(taper);
1331             continue_port_dumps();
1332             break;
1333
1334         default:
1335             error("driver received unexpected token (%s) from taper",
1336                   cmdstr[cmd]);
1337             /*NOTREACHED*/
1338         }
1339         /*
1340          * Wakeup any dumpers that are sleeping because of network
1341          * or disk constraints.
1342          */
1343         start_some_dumps(&runq);
1344
1345     } while(areads_dataready(taper));
1346 }
1347
1348 static dumper_t *
1349 idle_dumper(void)
1350 {
1351     dumper_t *dumper;
1352
1353     for(dumper = dmptable; dumper < dmptable+inparallel; dumper++)
1354         if(!dumper->busy && !dumper->down) return dumper;
1355
1356     return NULL;
1357 }
1358
1359 static void
1360 dumper_result(
1361     disk_t *    dp)
1362 {
1363     dumper_t *dumper;
1364     chunker_t *chunker;
1365     assignedhd_t **h=NULL;
1366     int activehd, i;
1367     off_t dummy;
1368     off_t size;
1369     int is_partial;
1370
1371     dumper = sched(dp)->dumper;
1372     chunker = dumper->chunker;
1373
1374     free_serial_dp(dp);
1375
1376     h = sched(dp)->holdp;
1377     activehd = sched(dp)->activehd;
1378
1379     if(dumper->result == DONE && chunker->result == DONE) {
1380         update_info_dumper(dp, sched(dp)->origsize,
1381                            sched(dp)->dumpsize, sched(dp)->dumptime);
1382         log_add(L_STATS, "estimate %s %s %s %d [sec %ld nkb " OFF_T_FMT
1383                 " ckb " OFF_T_FMT " kps %lu]",
1384                 dp->host->hostname, dp->name, sched(dp)->datestamp,
1385                 sched(dp)->level,
1386                 sched(dp)->est_time, (OFF_T_FMT_TYPE)sched(dp)->est_nsize,
1387                 (OFF_T_FMT_TYPE)sched(dp)->est_csize,
1388                 sched(dp)->est_kps);
1389     }
1390
1391     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
1392
1393     is_partial = dumper->result != DONE || chunker->result != DONE;
1394     rename_tmp_holding(sched(dp)->destname, !is_partial);
1395
1396     dummy = (off_t)0;
1397     for( i = 0, h = sched(dp)->holdp; i < activehd; i++ ) {
1398         dummy += h[i]->used;
1399     }
1400
1401     size = size_holding_files(sched(dp)->destname, 0);
1402     h[activehd]->used = size - dummy;
1403     holdalloc(h[activehd]->disk)->allocated_dumpers--;
1404     adjust_diskspace(dp, DONE);
1405
1406     sched(dp)->attempted += 1;
1407
1408     if((dumper->result != DONE || chunker->result != DONE) &&
1409        sched(dp)->attempted <= 1) {
1410         delete_diskspace(dp);
1411         enqueue_disk(&runq, dp);
1412     }
1413     else if(size > (off_t)DISK_BLOCK_KB) {
1414         sched(dp)->attempted = 0;
1415         enqueue_disk(&tapeq, dp);
1416         startaflush();
1417     }
1418     else {
1419         delete_diskspace(dp);
1420     }
1421
1422     dumper->busy = 0;
1423     dp->host->inprogress -= 1;
1424     dp->inprogress = 0;
1425
1426     waitpid(chunker->pid, NULL, 0 );
1427     aclose(chunker->fd);
1428     chunker->fd = -1;
1429     chunker->down = 1;
1430
1431     dp = NULL;
1432     if (chunker->result == ABORT_FINISHED)
1433         pending_aborts--;
1434     continue_port_dumps();
1435     /*
1436      * Wakeup any dumpers that are sleeping because of network
1437      * or disk constraints.
1438      */
1439     start_some_dumps(&runq);
1440 }
1441
1442
1443 static void
1444 handle_dumper_result(
1445     void *      cookie)
1446 {
1447     /*static int pending_aborts = 0;*/
1448     dumper_t *dumper = cookie;
1449     disk_t *dp, *sdp;
1450     cmd_t cmd;
1451     int result_argc;
1452     char *qname;
1453     char *result_argv[MAX_ARGS+1];
1454
1455     assert(dumper != NULL);
1456     dp = dumper->dp;
1457     assert(dp != NULL && sched(dp) != NULL);
1458
1459     do {
1460
1461         short_dump_state();
1462
1463         cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
1464
1465         if(cmd != BOGUS) {
1466             /* result_argv[2] always contains the serial number */
1467             sdp = serial2disk(result_argv[2]);
1468             if (sdp != dp) {
1469                 error("%s: Invalid serial number: %s", get_pname(), result_argv[2]);
1470                 /*NOTREACHED*/
1471             }
1472         }
1473
1474         qname = quote_string(dp->name);
1475         switch(cmd) {
1476
1477         case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
1478             if(result_argc != 6) {
1479                 error("error [dumper DONE result_argc != 6: %d]", result_argc);
1480                 /*NOTREACHED*/
1481             }
1482
1483             /*free_serial(result_argv[2]);*/
1484
1485             sched(dp)->origsize = OFF_T_ATOI(result_argv[3]);
1486             sched(dp)->dumptime = TIME_T_ATOI(result_argv[5]);
1487
1488             printf("driver: finished-cmd time %s %s dumped %s:%s\n",
1489                    walltime_str(curclock()), dumper->name,
1490                    dp->host->hostname, qname);
1491             fflush(stdout);
1492
1493             dumper->result = cmd;
1494
1495             break;
1496
1497         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
1498             /*
1499              * Requeue this disk, and fall through to the FAILED
1500              * case for cleanup.
1501              */
1502             if(sched(dp)->attempted) {
1503                 log_add(L_FAIL, "%s %s %s %d [too many dumper retry: %s]",
1504                     dp->host->hostname, dp->name, sched(dp)->datestamp,
1505                     sched(dp)->level, result_argv[3]);
1506                 printf("driver: dump failed %s %s %s, too many dumper retry: %s\n",
1507                         result_argv[2], dp->host->hostname, dp->name,
1508                         result_argv[3]);
1509             }
1510             /* FALLTHROUGH */
1511         case FAILED: /* FAILED <handle> <errstr> */
1512             /*free_serial(result_argv[2]);*/
1513             dumper->result = cmd;
1514             break;
1515
1516         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
1517             /*
1518              * We sent an ABORT from the NO-ROOM case because this dump
1519              * wasn't going to fit onto the holding disk.  We now need to
1520              * clean up the remains of this image, and try to finish
1521              * other dumps that are waiting on disk space.
1522              */
1523             assert(pending_aborts);
1524             /*free_serial(result_argv[2]);*/
1525             dumper->result = cmd;
1526             break;
1527
1528         case BOGUS:
1529             /* either EOF or garbage from dumper.  Turn it off */
1530             log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
1531                     dumper->name, (long)dumper->pid);
1532             if (dumper->ev_read) {
1533                 event_release(dumper->ev_read);
1534                 dumper->ev_read = NULL;
1535             }
1536             aclose(dumper->fd);
1537             dumper->busy = 0;
1538             dumper->down = 1;   /* mark it down so it isn't used again */
1539             if(dp) {
1540                 /* if it was dumping something, zap it and try again */
1541                 if(sched(dp)->attempted) {
1542                 log_add(L_FAIL, "%s %s %s %d [%s died]",
1543                         dp->host->hostname, qname, sched(dp)->datestamp,
1544                         sched(dp)->level, dumper->name);
1545                 }
1546                 else {
1547                 log_add(L_WARNING, "%s died while dumping %s:%s lev %d.",
1548                         dumper->name, dp->host->hostname, qname,
1549                         sched(dp)->level);
1550                 }
1551             }
1552             dumper->result = cmd;
1553             break;
1554
1555         default:
1556             assert(0);
1557         }
1558         amfree(qname);
1559
1560         /* send the dumper result to the chunker */
1561         if(dumper->chunker->down == 0 && dumper->chunker->fd != -1 &&
1562            dumper->chunker->result == LAST_TOK) {
1563             if(cmd == DONE) {
1564                 chunker_cmd(dumper->chunker, DONE, dp);
1565             }
1566             else {
1567                 chunker_cmd(dumper->chunker, FAILED, dp);
1568             }
1569         }
1570
1571         if(dumper->result != LAST_TOK && dumper->chunker->result != LAST_TOK)
1572             dumper_result(dp);
1573
1574     } while(areads_dataready(dumper->fd));
1575 }
1576
1577
1578 static void
1579 handle_chunker_result(
1580     void *      cookie)
1581 {
1582     /*static int pending_aborts = 0;*/
1583     chunker_t *chunker = cookie;
1584     assignedhd_t **h=NULL;
1585     dumper_t *dumper;
1586     disk_t *dp, *sdp;
1587     cmd_t cmd;
1588     int result_argc;
1589     char *result_argv[MAX_ARGS+1];
1590     int dummy;
1591     int activehd = -1;
1592     char *qname;
1593
1594     assert(chunker != NULL);
1595     dumper = chunker->dumper;
1596     assert(dumper != NULL);
1597     dp = dumper->dp;
1598     assert(dp != NULL);
1599     assert(sched(dp) != NULL);
1600     assert(sched(dp)->destname != NULL);
1601     assert(dp != NULL && sched(dp) != NULL && sched(dp)->destname);
1602
1603     if(dp && sched(dp) && sched(dp)->holdp) {
1604         h = sched(dp)->holdp;
1605         activehd = sched(dp)->activehd;
1606     }
1607
1608     do {
1609
1610         short_dump_state();
1611
1612         cmd = getresult(chunker->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
1613
1614         if(cmd != BOGUS) {
1615             /* result_argv[2] always contains the serial number */
1616             sdp = serial2disk(result_argv[2]);
1617             if (sdp != dp) {
1618                 error("%s: Invalid serial number: %s", get_pname(), result_argv[2]);
1619                 /*NOTREACHED*/
1620             }
1621         }
1622
1623         switch(cmd) {
1624
1625         case PARTIAL: /* PARTIAL <handle> <dumpsize> <errstr> */
1626         case DONE: /* DONE <handle> <dumpsize> <errstr> */
1627             if(result_argc != 4) {
1628                 error("error [chunker %s result_argc != 4: %d]", cmdstr[cmd],
1629                       result_argc);
1630                 /*NOTREACHED*/
1631             }
1632             /*free_serial(result_argv[2]);*/
1633
1634             sched(dp)->dumpsize = (off_t)atof(result_argv[3]);
1635
1636             qname = quote_string(dp->name);
1637             printf("driver: finished-cmd time %s %s chunked %s:%s\n",
1638                    walltime_str(curclock()), chunker->name,
1639                    dp->host->hostname, qname);
1640             fflush(stdout);
1641             amfree(qname);
1642
1643             event_release(chunker->ev_read);
1644
1645             chunker->result = cmd;
1646
1647             break;
1648
1649         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
1650             event_release(chunker->ev_read);
1651
1652             chunker->result = cmd;
1653
1654             break;
1655         case FAILED: /* FAILED <handle> <errstr> */
1656             /*free_serial(result_argv[2]);*/
1657
1658             event_release(chunker->ev_read);
1659
1660             chunker->result = cmd;
1661
1662             break;
1663
1664         case NO_ROOM: /* NO-ROOM <handle> <missing_size> */
1665             if (!h || activehd < 0) { /* should never happen */
1666                 error("!h || activehd < 0");
1667                 /*NOTREACHED*/
1668             }
1669             h[activehd]->used -= OFF_T_ATOI(result_argv[3]);
1670             h[activehd]->reserved -= OFF_T_ATOI(result_argv[3]);
1671             holdalloc(h[activehd]->disk)->allocated_space -= OFF_T_ATOI(result_argv[3]);
1672             h[activehd]->disk->disksize -= OFF_T_ATOI(result_argv[3]);
1673             break;
1674
1675         case RQ_MORE_DISK: /* RQ-MORE-DISK <handle> */
1676             if (!h || activehd < 0) { /* should never happen */
1677                 error("!h || activehd < 0");
1678                 /*NOTREACHED*/
1679             }
1680             holdalloc(h[activehd]->disk)->allocated_dumpers--;
1681             h[activehd]->used = h[activehd]->reserved;
1682             if( h[++activehd] ) { /* There's still some allocated space left.
1683                                    * Tell the dumper about it. */
1684                 sched(dp)->activehd++;
1685                 chunker_cmd( chunker, CONTINUE, dp );
1686             } else { /* !h[++activehd] - must allocate more space */
1687                 sched(dp)->act_size = sched(dp)->est_size; /* not quite true */
1688                 sched(dp)->est_size = (sched(dp)->act_size/(off_t)20) * (off_t)21; /* +5% */
1689                 sched(dp)->est_size = am_round(sched(dp)->est_size, (off_t)DISK_BLOCK_KB);
1690                 if (sched(dp)->est_size < sched(dp)->act_size + 2*DISK_BLOCK_KB)
1691                     sched(dp)->est_size += 2 * DISK_BLOCK_KB;
1692                 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1693                                     &dummy,
1694                                     h[activehd-1] );
1695                 if( !h ) {
1696                     /* No diskspace available. The reason for this will be
1697                      * determined in continue_port_dumps(). */
1698                     enqueue_disk( &roomq, dp );
1699                     continue_port_dumps();
1700                 } else {
1701                     /* OK, allocate space for disk and have chunker continue */
1702                     sched(dp)->activehd = assign_holdingdisk( h, dp );
1703                     chunker_cmd( chunker, CONTINUE, dp );
1704                     amfree(h);
1705                 }
1706             }
1707             break;
1708
1709         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
1710             /*
1711              * We sent an ABORT from the NO-ROOM case because this dump
1712              * wasn't going to fit onto the holding disk.  We now need to
1713              * clean up the remains of this image, and try to finish
1714              * other dumps that are waiting on disk space.
1715              */
1716             /*assert(pending_aborts);*/
1717
1718             /*free_serial(result_argv[2]);*/
1719
1720             event_release(chunker->ev_read);
1721
1722             chunker->result = cmd;
1723
1724             break;
1725
1726         case BOGUS:
1727             /* either EOF or garbage from chunker.  Turn it off */
1728             log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
1729                     chunker->name, (long)chunker->pid);
1730
1731             if(dp) {
1732                 /* if it was dumping something, zap it and try again */
1733                 if (!h || activehd < 0) { /* should never happen */
1734                     error("!h || activehd < 0");
1735                     /*NOTREACHED*/
1736                 }
1737                 qname = quote_string(dp->name);
1738                 if(sched(dp)->attempted) {
1739                     log_add(L_FAIL, "%s %s %s %d [%s died]",
1740                             dp->host->hostname, qname, sched(dp)->datestamp,
1741                             sched(dp)->level, chunker->name);
1742                 }
1743                 else {
1744                     log_add(L_WARNING, "%s died while dumping %s:%s lev %d.",
1745                             chunker->name, dp->host->hostname, qname,
1746                             sched(dp)->level);
1747                 }
1748                 amfree(qname);
1749                 dp = NULL;
1750             }
1751
1752             event_release(chunker->ev_read);
1753
1754             chunker->result = cmd;
1755
1756             break;
1757
1758         default:
1759             assert(0);
1760         }
1761
1762         if(chunker->result != LAST_TOK && chunker->dumper->result != LAST_TOK)
1763             dumper_result(dp);
1764
1765     } while(areads_dataready(chunker->fd));
1766 }
1767
1768
1769 static disklist_t
1770 read_flush(void)
1771 {
1772     sched_t *sp;
1773     disk_t *dp;
1774     int line;
1775     dumpfile_t file;
1776     char *hostname, *diskname, *datestamp;
1777     int level;
1778     char *destname;
1779     disk_t *dp1;
1780     char *inpline = NULL;
1781     char *command;
1782     char *s;
1783     int ch;
1784     disklist_t tq;
1785     char *qname = NULL;
1786
1787     tq.head = tq.tail = NULL;
1788
1789     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
1790         line++;
1791         if (inpline[0] == '\0')
1792             continue;
1793
1794         s = inpline;
1795         ch = *s++;
1796
1797         skip_whitespace(s, ch);                 /* find the command */
1798         if(ch == '\0') {
1799             error("flush line %d: syntax error (no command)", line);
1800             /*NOTREACHED*/
1801         }
1802         command = s - 1;
1803         skip_non_whitespace(s, ch);
1804         s[-1] = '\0';
1805
1806         if(strcmp(command,"ENDFLUSH") == 0) {
1807             break;
1808         }
1809
1810         if(strcmp(command,"FLUSH") != 0) {
1811             error("flush line %d: syntax error (%s != FLUSH)", line, command);
1812             /*NOTREACHED*/
1813         }
1814
1815         skip_whitespace(s, ch);                 /* find the hostname */
1816         if(ch == '\0') {
1817             error("flush line %d: syntax error (no hostname)", line);
1818             /*NOTREACHED*/
1819         }
1820         hostname = s - 1;
1821         skip_non_whitespace(s, ch);
1822         s[-1] = '\0';
1823
1824         skip_whitespace(s, ch);                 /* find the diskname */
1825         if(ch == '\0') {
1826             error("flush line %d: syntax error (no diskname)", line);
1827             /*NOTREACHED*/
1828         }
1829         qname = s - 1;
1830         skip_quoted_string(s, ch);
1831         s[-1] = '\0';                           /* terminate the disk name */
1832         diskname = unquote_string(qname);
1833
1834         skip_whitespace(s, ch);                 /* find the datestamp */
1835         if(ch == '\0') {
1836             error("flush line %d: syntax error (no datestamp)", line);
1837             /*NOTREACHED*/
1838         }
1839         datestamp = s - 1;
1840         skip_non_whitespace(s, ch);
1841         s[-1] = '\0';
1842
1843         skip_whitespace(s, ch);                 /* find the level number */
1844         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
1845             error("flush line %d: syntax error (bad level)", line);
1846             /*NOTREACHED*/
1847         }
1848         skip_integer(s, ch);
1849
1850         skip_whitespace(s, ch);                 /* find the filename */
1851         if(ch == '\0') {
1852             error("flush line %d: syntax error (no filename)", line);
1853             /*NOTREACHED*/
1854         }
1855         destname = s - 1;
1856         skip_non_whitespace(s, ch);
1857         s[-1] = '\0';
1858
1859         get_dumpfile(destname, &file);
1860         if( file.type != F_DUMPFILE) {
1861             if( file.type != F_CONT_DUMPFILE )
1862                 log_add(L_INFO, "%s: ignoring cruft file.", destname);
1863             amfree(diskname);
1864             continue;
1865         }
1866
1867         if(strcmp(hostname, file.name) != 0 ||
1868            strcmp(diskname, file.disk) != 0 ||
1869            strcmp(datestamp, file.datestamp) != 0) {
1870             log_add(L_INFO, "disk %s:%s not consistent with file %s",
1871                     hostname, diskname, destname);
1872             amfree(diskname);
1873             continue;
1874         }
1875         amfree(diskname);
1876
1877         dp = lookup_disk(file.name, file.disk);
1878
1879         if (dp == NULL) {
1880             log_add(L_INFO, "%s: disk %s:%s not in database, skipping it.",
1881                     destname, file.name, file.disk);
1882             continue;
1883         }
1884
1885         if(file.dumplevel < 0 || file.dumplevel > 9) {
1886             log_add(L_INFO, "%s: ignoring file with bogus dump level %d.",
1887                     destname, file.dumplevel);
1888             continue;
1889         }
1890
1891         dp1 = (disk_t *)alloc(SIZEOF(disk_t));
1892         *dp1 = *dp;
1893         dp1->next = dp1->prev = NULL;
1894
1895         /* add it to the flushhost list */
1896         if(!flushhost) {
1897             flushhost = alloc(SIZEOF(am_host_t));
1898             flushhost->next = NULL;
1899             flushhost->hostname = stralloc("FLUSHHOST");
1900             flushhost->up = NULL;
1901             flushhost->features = NULL;
1902         }
1903         dp1->hostnext = flushhost->disks;
1904         flushhost->disks = dp1;
1905
1906         sp = (sched_t *) alloc(SIZEOF(sched_t));
1907         sp->destname = stralloc(destname);
1908         sp->level = file.dumplevel;
1909         sp->dumpdate = NULL;
1910         sp->degr_dumpdate = NULL;
1911         sp->datestamp = stralloc(file.datestamp);
1912         sp->est_nsize = (off_t)0;
1913         sp->est_csize = (off_t)0;
1914         sp->est_time = 0;
1915         sp->est_kps = 10;
1916         sp->priority = 0;
1917         sp->degr_level = -1;
1918         sp->attempted = 0;
1919         sp->act_size = size_holding_files(destname, 0);
1920         sp->holdp = build_diskspace(destname);
1921         if(sp->holdp == NULL) continue;
1922         sp->dumper = NULL;
1923         sp->timestamp = (time_t)0;
1924
1925         dp1->up = (char *)sp;
1926
1927         enqueue_disk(&tq, dp1);
1928     }
1929     amfree(inpline);
1930
1931     /*@i@*/ return tq;
1932 }
1933
1934 static void
1935 read_schedule(
1936     void *      cookie)
1937 {
1938     sched_t *sp;
1939     disk_t *dp;
1940     int level, line, priority;
1941     char *dumpdate, *degr_dumpdate;
1942     int degr_level;
1943     time_t time, degr_time;
1944     time_t *time_p = &time;
1945     time_t *degr_time_p = &degr_time;
1946     off_t nsize, csize, degr_nsize, degr_csize;
1947     unsigned long kps, degr_kps;
1948     char *hostname, *features, *diskname, *datestamp, *inpline = NULL;
1949     char *command;
1950     char *s;
1951     int ch;
1952     off_t flush_size = (off_t)0;
1953     char *qname = NULL;
1954     OFF_T_FMT_TYPE nsize_;
1955     OFF_T_FMT_TYPE csize_;
1956     OFF_T_FMT_TYPE degr_nsize_;
1957     OFF_T_FMT_TYPE degr_csize_;
1958
1959     (void)cookie;       /* Quiet unused parameter warning */
1960
1961     event_release(schedule_ev_read);
1962
1963     /* read schedule from stdin */
1964
1965     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
1966         if (inpline[0] == '\0')
1967             continue;
1968         line++;
1969
1970         s = inpline;
1971         ch = *s++;
1972
1973         skip_whitespace(s, ch);                 /* find the command */
1974         if(ch == '\0') {
1975             error("schedule line %d: syntax error (no command)", line);
1976             /*NOTREACHED*/
1977         }
1978         command = s - 1;
1979         skip_non_whitespace(s, ch);
1980         s[-1] = '\0';
1981
1982         if(strcmp(command,"DUMP") != 0) {
1983             error("schedule line %d: syntax error (%s != DUMP)", line, command);
1984             /*NOTREACHED*/
1985         }
1986
1987         skip_whitespace(s, ch);                 /* find the host name */
1988         if(ch == '\0') {
1989             error("schedule line %d: syntax error (no host name)", line);
1990             /*NOTREACHED*/
1991         }
1992         hostname = s - 1;
1993         skip_non_whitespace(s, ch);
1994         s[-1] = '\0';
1995
1996         skip_whitespace(s, ch);                 /* find the feature list */
1997         if(ch == '\0') {
1998             error("schedule line %d: syntax error (no feature list)", line);
1999             /*NOTREACHED*/
2000         }
2001         features = s - 1;
2002         skip_non_whitespace(s, ch);
2003         s[-1] = '\0';
2004
2005         skip_whitespace(s, ch);                 /* find the disk name */
2006         if(ch == '\0') {
2007             error("schedule line %d: syntax error (no disk name)", line);
2008             /*NOTREACHED*/
2009         }
2010         qname = s - 1;
2011         skip_quoted_string(s, ch);
2012         s[-1] = '\0';                           /* terminate the disk name */
2013         diskname = unquote_string(qname);
2014
2015         skip_whitespace(s, ch);                 /* find the datestamp */
2016         if(ch == '\0') {
2017             error("schedule line %d: syntax error (no datestamp)", line);
2018             /*NOTREACHED*/
2019         }
2020         datestamp = s - 1;
2021         skip_non_whitespace(s, ch);
2022         s[-1] = '\0';
2023
2024         skip_whitespace(s, ch);                 /* find the priority number */
2025         if(ch == '\0' || sscanf(s - 1, "%d", &priority) != 1) {
2026             error("schedule line %d: syntax error (bad priority)", line);
2027             /*NOTREACHED*/
2028         }
2029         skip_integer(s, ch);
2030
2031         skip_whitespace(s, ch);                 /* find the level number */
2032         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2033             error("schedule line %d: syntax error (bad level)", line);
2034             /*NOTREACHED*/
2035         }
2036         skip_integer(s, ch);
2037
2038         skip_whitespace(s, ch);                 /* find the dump date */
2039         if(ch == '\0') {
2040             error("schedule line %d: syntax error (bad dump date)", line);
2041             /*NOTREACHED*/
2042         }
2043         dumpdate = s - 1;
2044         skip_non_whitespace(s, ch);
2045         s[-1] = '\0';
2046
2047         skip_whitespace(s, ch);                 /* find the native size */
2048         nsize_ = (OFF_T_FMT_TYPE)0;
2049         if(ch == '\0' || sscanf(s - 1, OFF_T_FMT, &nsize_) != 1) {
2050             error("schedule line %d: syntax error (bad nsize)", line);
2051             /*NOTREACHED*/
2052         }
2053         nsize = nsize_;
2054         skip_integer(s, ch);
2055
2056         skip_whitespace(s, ch);                 /* find the compressed size */
2057         csize_ = (OFF_T_FMT_TYPE)0;
2058         if(ch == '\0' || sscanf(s - 1, OFF_T_FMT, &csize_) != 1) {
2059             error("schedule line %d: syntax error (bad csize)", line);
2060             /*NOTREACHED*/
2061         }
2062         csize = csize_;
2063         skip_integer(s, ch);
2064
2065         skip_whitespace(s, ch);                 /* find the time number */
2066         if(ch == '\0' || sscanf(s - 1, TIME_T_FMT,
2067                                 (TIME_T_FMT_TYPE *)time_p) != 1) {
2068             error("schedule line %d: syntax error (bad estimated time)", line);
2069             /*NOTREACHED*/
2070         }
2071         skip_integer(s, ch);
2072
2073         skip_whitespace(s, ch);                 /* find the kps number */
2074         if(ch == '\0' || sscanf(s - 1, "%lu", &kps) != 1) {
2075             error("schedule line %d: syntax error (bad kps)", line);
2076             continue;
2077         }
2078         skip_integer(s, ch);
2079
2080         degr_dumpdate = NULL;                   /* flag if degr fields found */
2081         skip_whitespace(s, ch);                 /* find the degr level number */
2082         if(ch != '\0') {
2083             if(sscanf(s - 1, "%d", &degr_level) != 1) {
2084                 error("schedule line %d: syntax error (bad degr level)", line);
2085                 /*NOTREACHED*/
2086             }
2087             skip_integer(s, ch);
2088
2089             skip_whitespace(s, ch);             /* find the degr dump date */
2090             if(ch == '\0') {
2091                 error("schedule line %d: syntax error (bad degr dump date)", line);
2092                 /*NOTREACHED*/
2093             }
2094             degr_dumpdate = s - 1;
2095             skip_non_whitespace(s, ch);
2096             s[-1] = '\0';
2097
2098             skip_whitespace(s, ch);             /* find the degr native size */
2099             degr_nsize_ = (OFF_T_FMT_TYPE)0;
2100             if(ch == '\0'  || sscanf(s - 1, OFF_T_FMT, &degr_nsize_) != 1) {
2101                 error("schedule line %d: syntax error (bad degr nsize)", line);
2102                 /*NOTREACHED*/
2103             }
2104             degr_nsize = degr_nsize_;
2105             skip_integer(s, ch);
2106
2107             skip_whitespace(s, ch);             /* find the degr compressed size */
2108             degr_csize_ = (OFF_T_FMT_TYPE)0;
2109             if(ch == '\0'  || sscanf(s - 1, OFF_T_FMT, &degr_csize_) != 1) {
2110                 error("schedule line %d: syntax error (bad degr csize)", line);
2111                 /*NOTREACHED*/
2112             }
2113             degr_csize = degr_csize_;
2114             skip_integer(s, ch);
2115
2116             skip_whitespace(s, ch);             /* find the degr time number */
2117             if(ch == '\0' || sscanf(s - 1, TIME_T_FMT,
2118                                 (TIME_T_FMT_TYPE *)degr_time_p) != 1) {
2119                 error("schedule line %d: syntax error (bad degr estimated time)", line);
2120                 /*NOTREACHED*/
2121             }
2122             skip_integer(s, ch);
2123
2124             skip_whitespace(s, ch);             /* find the degr kps number */
2125             if(ch == '\0' || sscanf(s - 1, "%lu", &degr_kps) != 1) {
2126                 error("schedule line %d: syntax error (bad degr kps)", line);
2127                 /*NOTREACHED*/
2128             }
2129             skip_integer(s, ch);
2130         }
2131
2132         dp = lookup_disk(hostname, diskname);
2133         if(dp == NULL) {
2134             log_add(L_WARNING,
2135                     "schedule line %d: %s:'%s' not in disklist, ignored",
2136                     line, hostname, qname);
2137             amfree(diskname);
2138             continue;
2139         }
2140
2141         sp = (sched_t *) alloc(SIZEOF(sched_t));
2142         /*@ignore@*/
2143         sp->level = level;
2144         sp->dumpdate = stralloc(dumpdate);
2145         sp->est_nsize = DISK_BLOCK_KB + nsize; /* include header */
2146         sp->est_csize = DISK_BLOCK_KB + csize; /* include header */
2147         /* round estimate to next multiple of DISK_BLOCK_KB */
2148         sp->est_csize = am_round(sp->est_csize, DISK_BLOCK_KB);
2149         sp->est_size = sp->est_csize;
2150         sp->est_time = time;
2151         sp->est_kps = kps;
2152         sp->priority = priority;
2153         sp->datestamp = stralloc(datestamp);
2154
2155         if(degr_dumpdate) {
2156             sp->degr_level = degr_level;
2157             sp->degr_dumpdate = stralloc(degr_dumpdate);
2158             sp->degr_nsize = DISK_BLOCK_KB + degr_nsize;
2159             sp->degr_csize = DISK_BLOCK_KB + degr_csize;
2160             /* round estimate to next multiple of DISK_BLOCK_KB */
2161             sp->degr_csize = am_round(sp->degr_csize, DISK_BLOCK_KB);
2162             sp->degr_time = degr_time;
2163             sp->degr_kps = degr_kps;
2164         } else {
2165             sp->degr_level = -1;
2166             sp->degr_dumpdate = NULL;
2167         }
2168         /*@end@*/
2169
2170         sp->attempted = 0;
2171         sp->act_size = (off_t)0;
2172         sp->holdp = NULL;
2173         sp->activehd = -1;
2174         sp->dumper = NULL;
2175         sp->timestamp = (time_t)0;
2176         sp->destname = NULL;
2177         sp->no_space = 0;
2178
2179         dp->up = (char *) sp;
2180         if(dp->host->features == NULL) {
2181             dp->host->features = am_string_to_feature(features);
2182         }
2183         remove_disk(&waitq, dp);
2184         enqueue_disk(&runq, dp);
2185         flush_size += sp->act_size;
2186         amfree(diskname);
2187     }
2188     printf("driver: flush size " OFF_T_FMT "\n", (OFF_T_FMT_TYPE)flush_size);
2189     amfree(inpline);
2190     if(line == 0)
2191         log_add(L_WARNING, "WARNING: got empty schedule from planner");
2192     if(need_degraded==1) start_degraded_mode(&runq);
2193     start_some_dumps(&runq);
2194 }
2195
2196 static unsigned long
2197 free_kps(
2198     interface_t *ip)
2199 {
2200     unsigned long res;
2201
2202     if (ip == (interface_t *)0) {
2203         interface_t *p;
2204         unsigned long maxusage=0;
2205         unsigned long curusage=0;
2206         for(p = lookup_interface(NULL); p != NULL; p = p->next) {
2207             maxusage += interface_get_maxusage(p);
2208             curusage += p->curusage;
2209         }
2210         res = maxusage - curusage;
2211 #ifndef __lint
2212     } else {
2213         res = interface_get_maxusage(ip) - ip->curusage;
2214 #endif
2215     }
2216
2217     return res;
2218 }
2219
2220 static void
2221 interface_state(
2222     char *time_str)
2223 {
2224     interface_t *ip;
2225
2226     printf("driver: interface-state time %s", time_str);
2227
2228     for(ip = lookup_interface(NULL); ip != NULL; ip = ip->next) {
2229         printf(" if %s: free %lu", ip->name, free_kps(ip));
2230     }
2231     printf("\n");
2232 }
2233
2234 static void
2235 allocate_bandwidth(
2236     interface_t *       ip,
2237     unsigned long       kps)
2238 {
2239     ip->curusage += kps;
2240 }
2241
2242 static void
2243 deallocate_bandwidth(
2244     interface_t *       ip,
2245     unsigned long       kps)
2246 {
2247     assert(kps <= ip->curusage);
2248     ip->curusage -= kps;
2249 }
2250
2251 /* ------------ */
2252 static off_t
2253 free_space(void)
2254 {
2255     holdingdisk_t *hdp;
2256     off_t total_free;
2257     off_t diff;
2258
2259     total_free = (off_t)0;
2260     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2261         diff = hdp->disksize - holdalloc(hdp)->allocated_space;
2262         if(diff > (off_t)0)
2263             total_free += diff;
2264     }
2265     return total_free;
2266 }
2267
2268 /*
2269  * We return an array of pointers to assignedhd_t. The array contains at
2270  * most one entry per holding disk. The list of pointers is terminated by
2271  * a NULL pointer. Each entry contains a pointer to a holdingdisk and
2272  * how much diskspace to use on that disk. Later on, assign_holdingdisk
2273  * will allocate the given amount of space.
2274  * If there is not enough room on the holdingdisks, NULL is returned.
2275  */
2276
2277 static assignedhd_t **
2278 find_diskspace(
2279     off_t               size,
2280     int *               cur_idle,
2281     assignedhd_t *      pref)
2282 {
2283     assignedhd_t **result = NULL;
2284     holdingdisk_t *minp, *hdp;
2285     int i=0, num_holdingdisks=0; /* are we allowed to use the global thing? */
2286     int j, minj;
2287     char *used;
2288     off_t halloc, dalloc, hfree, dfree;
2289
2290     (void)cur_idle;     /* Quiet unused parameter warning */
2291
2292     if (size < 2*DISK_BLOCK_KB)
2293         size = 2*DISK_BLOCK_KB;
2294     size = am_round(size, (off_t)DISK_BLOCK_KB);
2295
2296 #ifdef HOLD_DEBUG
2297     printf("%s: want " OFF_T_FMT " K\n", debug_prefix_time(": find_diskspace"),
2298            (OFF_T_FMT_TYPE)size);
2299     fflush(stdout);
2300 #endif
2301
2302     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2303         num_holdingdisks++;
2304     }
2305
2306     used = alloc(SIZEOF(*used) * num_holdingdisks);/*disks used during this run*/
2307     memset( used, 0, (size_t)num_holdingdisks );
2308     result = alloc(SIZEOF(assignedhd_t *) * (num_holdingdisks + 1));
2309     result[0] = NULL;
2310
2311     while( i < num_holdingdisks && size > (off_t)0 ) {
2312         /* find the holdingdisk with the fewest active dumpers and among
2313          * those the one with the biggest free space
2314          */
2315         minp = NULL; minj = -1;
2316         for(j = 0, hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next, j++ ) {
2317             if( pref && pref->disk == hdp && !used[j] &&
2318                 holdalloc(hdp)->allocated_space <= hdp->disksize - (off_t)DISK_BLOCK_KB) {
2319                 minp = hdp;
2320                 minj = j;
2321                 break;
2322             }
2323             else if( holdalloc(hdp)->allocated_space <= hdp->disksize - (off_t)(2*DISK_BLOCK_KB) &&
2324                 !used[j] &&
2325                 (!minp ||
2326                  holdalloc(hdp)->allocated_dumpers < holdalloc(minp)->allocated_dumpers ||
2327                  (holdalloc(hdp)->allocated_dumpers == holdalloc(minp)->allocated_dumpers &&
2328                   hdp->disksize-holdalloc(hdp)->allocated_space > minp->disksize-holdalloc(minp)->allocated_space)) ) {
2329                 minp = hdp;
2330                 minj = j;
2331             }
2332         }
2333
2334         pref = NULL;
2335         if( !minp ) { break; } /* all holding disks are full */
2336         used[minj] = 1;
2337
2338         /* hfree = free space on the disk */
2339         hfree = minp->disksize - holdalloc(minp)->allocated_space;
2340
2341         /* dfree = free space for data, remove 1 header for each chunksize */
2342         dfree = hfree - (((hfree-(off_t)1)/holdingdisk_get_chunksize(minp))+(off_t)1) * (off_t)DISK_BLOCK_KB;
2343
2344         /* dalloc = space I can allocate for data */
2345         dalloc = ( dfree < size ) ? dfree : size;
2346
2347         /* halloc = space to allocate, including 1 header for each chunksize */
2348         halloc = dalloc + (((dalloc-(off_t)1)/holdingdisk_get_chunksize(minp))+(off_t)1) * (off_t)DISK_BLOCK_KB;
2349
2350 #ifdef HOLD_DEBUG
2351         printf("%s: find diskspace: size " OFF_T_FMT " hf " OFF_T_FMT
2352                " df " OFF_T_FMT " da " OFF_T_FMT " ha " OFF_T_FMT "\n",
2353                debug_prefix_time(": find_diskspace"),
2354                (OFF_T_FMT_TYPE)size,
2355                (OFF_T_FMT_TYPE)hfree,
2356                (OFF_T_FMT_TYPE)dfree,
2357                (OFF_T_FMT_TYPE)dalloc,
2358                (OFF_T_FMT_TYPE)halloc);
2359         fflush(stdout);
2360 #endif
2361         size -= dalloc;
2362         result[i] = alloc(SIZEOF(assignedhd_t));
2363         result[i]->disk = minp;
2364         result[i]->reserved = halloc;
2365         result[i]->used = (off_t)0;
2366         result[i]->destname = NULL;
2367         result[i+1] = NULL;
2368         i++;
2369     } /* while i < num_holdingdisks && size > 0 */
2370     amfree(used);
2371
2372     if(size != (off_t)0) { /* not enough space available */
2373         printf("find diskspace: not enough diskspace. Left with "
2374                OFF_T_FMT " K\n", (OFF_T_FMT_TYPE)size);
2375         fflush(stdout);
2376         free_assignedhd(result);
2377         result = NULL;
2378     }
2379
2380 #ifdef HOLD_DEBUG
2381     for( i = 0; result && result[i]; i++ ) {
2382         printf("%s: find diskspace: selected %s free " OFF_T_FMT " reserved " OFF_T_FMT " dumpers %d\n",
2383                 debug_prefix_time(": find_diskspace"),
2384                 holdingdisk_get_diskdir(result[i]->disk),
2385                 (OFF_T_FMT_TYPE)(result[i]->disk->disksize -
2386                   holdalloc(result[i]->disk)->allocated_space),
2387                 (OFF_T_FMT_TYPE)result[i]->reserved,
2388                 holdalloc(result[i]->disk)->allocated_dumpers);
2389     }
2390     fflush(stdout);
2391 #endif
2392
2393     return result;
2394 }
2395
2396 static int
2397 assign_holdingdisk(
2398     assignedhd_t **     holdp,
2399     disk_t *            diskp)
2400 {
2401     int i, j, c, l=0;
2402     off_t size;
2403     char *sfn = sanitise_filename(diskp->name);
2404     char lvl[64];
2405     assignedhd_t **new_holdp;
2406     char *qname;
2407
2408     snprintf( lvl, SIZEOF(lvl), "%d", sched(diskp)->level );
2409
2410     size = am_round(sched(diskp)->est_size - sched(diskp)->act_size,
2411                     (off_t)DISK_BLOCK_KB);
2412
2413     for( c = 0; holdp[c]; c++ )
2414         (void)c; /* count number of disks */
2415
2416     /* allocate memory for sched(diskp)->holdp */
2417     for(j = 0; sched(diskp)->holdp && sched(diskp)->holdp[j]; j++)
2418         (void)j;        /* Quiet lint */
2419     new_holdp = (assignedhd_t **)alloc(SIZEOF(assignedhd_t*)*(j+c+1));
2420     if (sched(diskp)->holdp) {
2421         memcpy(new_holdp, sched(diskp)->holdp, j * SIZEOF(*new_holdp));
2422         amfree(sched(diskp)->holdp);
2423     }
2424     sched(diskp)->holdp = new_holdp;
2425     new_holdp = NULL;
2426
2427     i = 0;
2428     if( j > 0 ) { /* This is a request for additional diskspace. See if we can
2429                    * merge assignedhd_t's */
2430         l=j;
2431         if( sched(diskp)->holdp[j-1]->disk == holdp[0]->disk ) { /* Yes! */
2432             sched(diskp)->holdp[j-1]->reserved += holdp[0]->reserved;
2433             holdalloc(holdp[0]->disk)->allocated_space += holdp[0]->reserved;
2434             size = (holdp[0]->reserved>size) ? (off_t)0 : size-holdp[0]->reserved;
2435             qname = quote_string(diskp->name);
2436 #ifdef HOLD_DEBUG
2437             printf("%s: merging holding disk %s to disk %s:%s, add " OFF_T_FMT " for reserved " OFF_T_FMT ", left " OFF_T_FMT "\n",
2438                    debug_prefix_time(": assign_holdingdisk"),
2439                    holdingdisk_get_diskdir(sched(diskp)->holdp[j-1]->disk),
2440                    diskp->host->hostname, qname,
2441                    (OFF_T_FMT_TYPE)holdp[0]->reserved,
2442                    (OFF_T_FMT_TYPE)sched(diskp)->holdp[j-1]->reserved,
2443                    (OFF_T_FMT_TYPE)size);
2444             fflush(stdout);
2445 #endif
2446             i++;
2447             amfree(qname);
2448             amfree(holdp[0]);
2449             l=j-1;
2450         }
2451     }
2452
2453     /* copy assignedhd_s to sched(diskp), adjust allocated_space */
2454     for( ; holdp[i]; i++ ) {
2455         holdp[i]->destname = newvstralloc( holdp[i]->destname,
2456                                            holdingdisk_get_diskdir(holdp[i]->disk), "/",
2457                                            hd_driver_timestamp, "/",
2458                                            diskp->host->hostname, ".",
2459                                            sfn, ".",
2460                                            lvl, NULL );
2461         sched(diskp)->holdp[j++] = holdp[i];
2462         holdalloc(holdp[i]->disk)->allocated_space += holdp[i]->reserved;
2463         size = (holdp[i]->reserved > size) ? (off_t)0 :
2464                   (size - holdp[i]->reserved);
2465         qname = quote_string(diskp->name);
2466 #ifdef HOLD_DEBUG
2467         printf("%s: %d assigning holding disk %s to disk %s:%s, reserved " OFF_T_FMT ", left " OFF_T_FMT "\n",
2468                 debug_prefix_time(": assign_holdingdisk"),
2469                 i, holdingdisk_get_diskdir(holdp[i]->disk), diskp->host->hostname, qname,
2470                 (OFF_T_FMT_TYPE)holdp[i]->reserved,
2471                 (OFF_T_FMT_TYPE)size);
2472         fflush(stdout);
2473 #endif
2474         amfree(qname);
2475         holdp[i] = NULL; /* so it doesn't get free()d... */
2476     }
2477     sched(diskp)->holdp[j] = NULL;
2478     amfree(sfn);
2479
2480     return l;
2481 }
2482
2483 static void
2484 adjust_diskspace(
2485     disk_t *    diskp,
2486     cmd_t       cmd)
2487 {
2488     assignedhd_t **holdp;
2489     off_t total = (off_t)0;
2490     off_t diff;
2491     int i;
2492     char *qname, *hqname, *qdest;
2493
2494     (void)cmd;  /* Quiet unused parameter warning */
2495
2496     qname = quote_string(diskp->name);
2497     qdest = quote_string(sched(diskp)->destname);
2498 #ifdef HOLD_DEBUG
2499     printf("%s: %s:%s %s\n",
2500            debug_prefix_time(": adjust_diskspace"),
2501            diskp->host->hostname, qname, qdest);
2502     fflush(stdout);
2503 #endif
2504
2505     holdp = sched(diskp)->holdp;
2506
2507     assert(holdp != NULL);
2508
2509     for( i = 0; holdp[i]; i++ ) { /* for each allocated disk */
2510         diff = holdp[i]->used - holdp[i]->reserved;
2511         total += holdp[i]->used;
2512         holdalloc(holdp[i]->disk)->allocated_space += diff;
2513         hqname = quote_string(holdp[i]->disk->name);
2514 #ifdef HOLD_DEBUG
2515         printf("%s: hdisk %s done, reserved " OFF_T_FMT " used " OFF_T_FMT " diff " OFF_T_FMT " alloc " OFF_T_FMT " dumpers %d\n",
2516                 debug_prefix_time(": adjust_diskspace"),
2517                 holdp[i]->disk->name,
2518                 (OFF_T_FMT_TYPE)holdp[i]->reserved,
2519                 (OFF_T_FMT_TYPE)holdp[i]->used,
2520                 (OFF_T_FMT_TYPE)diff,
2521                 (OFF_T_FMT_TYPE)holdalloc(holdp[i]->disk)->allocated_space,
2522                 holdalloc(holdp[i]->disk)->allocated_dumpers );
2523         fflush(stdout);
2524 #endif
2525         holdp[i]->reserved += diff;
2526         amfree(hqname);
2527     }
2528
2529     sched(diskp)->act_size = total;
2530
2531 #ifdef HOLD_DEBUG
2532     printf("%s: after: disk %s:%s used " OFF_T_FMT "\n",
2533            debug_prefix_time(": adjust_diskspace"),
2534            diskp->host->hostname, qname,
2535            (OFF_T_FMT_TYPE)sched(diskp)->act_size);
2536     fflush(stdout);
2537 #endif
2538     amfree(qdest);
2539     amfree(qname);
2540 }
2541
2542 static void
2543 delete_diskspace(
2544     disk_t *diskp)
2545 {
2546     assignedhd_t **holdp;
2547     int i;
2548
2549     holdp = sched(diskp)->holdp;
2550
2551     assert(holdp != NULL);
2552
2553     for( i = 0; holdp[i]; i++ ) { /* for each disk */
2554         /* find all files of this dump on that disk, and subtract their
2555          * reserved sizes from the disk's allocated space
2556          */
2557         holdalloc(holdp[i]->disk)->allocated_space -= holdp[i]->used;
2558     }
2559
2560     unlink_holding_files(holdp[0]->destname);   /* no need for the entire list,
2561                                                  * because unlink_holding_files
2562                                                  * will walk through all files
2563                                                  * using cont_filename */
2564     free_assignedhd(sched(diskp)->holdp);
2565     sched(diskp)->holdp = NULL;
2566     sched(diskp)->act_size = (off_t)0;
2567 }
2568
2569 static assignedhd_t **
2570 build_diskspace(
2571     char *      destname)
2572 {
2573     int i, j;
2574     int fd;
2575     ssize_t buflen;
2576     char buffer[DISK_BLOCK_BYTES];
2577     dumpfile_t file;
2578     assignedhd_t **result;
2579     holdingdisk_t *hdp;
2580     off_t *used;
2581     int num_holdingdisks=0;
2582     char dirname[1000], *ch;
2583     struct stat finfo;
2584     char *filename = destname;
2585
2586     memset(buffer, 0, sizeof(buffer));
2587     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2588         num_holdingdisks++;
2589     }
2590     used = alloc(SIZEOF(off_t) * num_holdingdisks);
2591     for(i=0;i<num_holdingdisks;i++)
2592         used[i] = (off_t)0;
2593     result = alloc(SIZEOF(assignedhd_t *) * (num_holdingdisks + 1));
2594     result[0] = NULL;
2595     while(filename != NULL && filename[0] != '\0') {
2596         strncpy(dirname, filename, 999);
2597         dirname[999]='\0';
2598         ch = strrchr(dirname,'/');
2599         *ch = '\0';
2600         ch = strrchr(dirname,'/');
2601         *ch = '\0';
2602
2603         for(j = 0, hdp = getconf_holdingdisks(); hdp != NULL;
2604                                                  hdp = hdp->next, j++ ) {
2605             if(strcmp(dirname, holdingdisk_get_diskdir(hdp))==0) {
2606                 break;
2607             }
2608         }
2609
2610         if(stat(filename, &finfo) == -1) {
2611             fprintf(stderr, "stat %s: %s\n", filename, strerror(errno));
2612             finfo.st_size = (off_t)0;
2613         }
2614         used[j] += ((off_t)finfo.st_size+(off_t)1023)/(off_t)1024;
2615         if((fd = open(filename,O_RDONLY)) == -1) {
2616             fprintf(stderr,"build_diskspace: open of %s failed: %s\n",
2617                     filename, strerror(errno));
2618             return NULL;
2619         }
2620         if ((buflen = fullread(fd, buffer, SIZEOF(buffer))) > 0) {;
2621                 parse_file_header(buffer, &file, (size_t)buflen);
2622         }
2623         close(fd);
2624         filename = file.cont_filename;
2625     }
2626
2627     for(j = 0, i=0, hdp = getconf_holdingdisks(); hdp != NULL;
2628                                                   hdp = hdp->next, j++ ) {
2629         if(used[j] != (off_t)0) {
2630             result[i] = alloc(SIZEOF(assignedhd_t));
2631             result[i]->disk = hdp;
2632             result[i]->reserved = used[j];
2633             result[i]->used = used[j];
2634             result[i]->destname = stralloc(destname);
2635             result[i+1] = NULL;
2636             i++;
2637         }
2638     }
2639
2640     amfree(used);
2641     return result;
2642 }
2643
2644 static void
2645 holdingdisk_state(
2646     char *      time_str)
2647 {
2648     holdingdisk_t *hdp;
2649     int dsk;
2650     off_t diff;
2651
2652     printf("driver: hdisk-state time %s", time_str);
2653
2654     for(hdp = getconf_holdingdisks(), dsk = 0; hdp != NULL; hdp = hdp->next, dsk++) {
2655         diff = hdp->disksize - holdalloc(hdp)->allocated_space;
2656         printf(" hdisk %d: free " OFF_T_FMT " dumpers %d", dsk,
2657                (OFF_T_FMT_TYPE)diff, holdalloc(hdp)->allocated_dumpers);
2658     }
2659     printf("\n");
2660 }
2661
2662 static void
2663 update_failed_dump_to_tape(
2664     disk_t *    dp)
2665 {
2666 /* JLM
2667  * should simply set no_bump
2668  */
2669
2670     time_t save_timestamp = sched(dp)->timestamp;
2671     /* setting timestamp to 0 removes the current level from the
2672      * database, so that we ensure that it will not be bumped to the
2673      * next level on the next run.  If we didn't do this, dumpdates or
2674      * gnutar-lists might have been updated already, and a bumped
2675      * incremental might be created.  */
2676     sched(dp)->timestamp = 0;
2677     update_info_dumper(dp, (off_t)-1, (off_t)-1, (time_t)-1);
2678     sched(dp)->timestamp = save_timestamp;
2679 }
2680
2681 /* ------------------- */
2682 static int
2683 dump_to_tape(
2684     disk_t *    dp)
2685 {
2686     dumper_t *dumper;
2687     int failed = 0;
2688     off_t filenum;
2689     off_t origsize = (off_t)0;
2690     off_t dumpsize = (off_t)0;
2691     time_t dumptime = (time_t)0;
2692     double tapetime = 0.0;
2693     cmd_t cmd;
2694     int result_argc, rc;
2695     char *result_argv[MAX_ARGS+1];
2696     int dumper_tryagain = 0;
2697     char *qname;
2698
2699     qname = quote_string(dp->name);
2700     printf("driver: dumping %s:%s directly to tape\n",
2701            dp->host->hostname, qname);
2702     fflush(stdout);
2703
2704     /* pick a dumper and fail if there are no idle dumpers */
2705
2706     dumper = idle_dumper();
2707     if (!dumper) {
2708         printf("driver: no idle dumpers for %s:%s.\n",
2709                 dp->host->hostname, qname);
2710         fflush(stdout);
2711         log_add(L_WARNING, "no idle dumpers for %s:%s.\n",
2712                 dp->host->hostname, qname);
2713         amfree(qname);
2714         return 2;       /* fatal problem */
2715     }
2716
2717     /* tell the taper to read from a port number of its choice */
2718
2719     taper_cmd(PORT_WRITE, dp, NULL, sched(dp)->level, sched(dp)->datestamp);
2720     cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
2721     if(cmd != PORT) {
2722         printf("driver: did not get PORT from taper for %s:%s\n",
2723                 dp->host->hostname, qname);
2724         fflush(stdout);
2725         amfree(qname);
2726         return 2;       /* fatal problem */
2727     }
2728     /* copy port number */
2729     dumper->output_port = atoi(result_argv[2]);
2730
2731     /* tell the dumper to dump to a port */
2732
2733     dumper_cmd(dumper, PORT_DUMP, dp);
2734     dp->host->start_t = time(NULL) + 15;
2735
2736     /* update statistics & print state */
2737
2738     taper_busy = dumper->busy = 1;
2739     dp->host->inprogress += 1;
2740     dp->inprogress = 1;
2741     sched(dp)->timestamp = time((time_t *)0);
2742     allocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2743     idle_reason = NOT_IDLE;
2744
2745     short_dump_state();
2746
2747     /* wait for result from dumper */
2748
2749     cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
2750
2751     switch(cmd) {
2752     case BOGUS:
2753         /* either eof or garbage from dumper */
2754         log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
2755                 dumper->name, (long)dumper->pid);
2756         dumper->down = 1;       /* mark it down so it isn't used again */
2757         failed = 1;     /* dump failed, must still finish up with taper */
2758         break;
2759
2760     case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
2761         /* everything went fine */
2762         origsize = (off_t)atof(result_argv[3]);
2763         /*dumpsize = (off_t)atof(result_argv[4]);*/
2764         dumptime = (time_t)atof(result_argv[5]);
2765         break;
2766
2767     case NO_ROOM: /* NO-ROOM <handle> */
2768         dumper_cmd(dumper, ABORT, dp);
2769         cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
2770         assert(cmd == ABORT_FINISHED);
2771
2772     case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2773     default:
2774         /* dump failed, but we must still finish up with taper */
2775         /* problem with dump, possibly nonfatal, retry one time */
2776         sched(dp)->attempted++;
2777         failed = sched(dp)->attempted;
2778         dumper_tryagain = 1;
2779         break;
2780
2781     case FAILED: /* FAILED <handle> <errstr> */
2782         /* dump failed, but we must still finish up with taper */
2783         failed = 2;     /* fatal problem with dump */
2784         break;
2785     }
2786
2787     /*
2788      * Note that at this point, even if the dump above failed, it may
2789      * not be a fatal failure if taper below says we can try again.
2790      * E.g. a dumper failure above may actually be the result of a
2791      * tape overflow, which in turn causes dump to see "broken pipe",
2792      * "no space on device", etc., since taper closed the port first.
2793      */
2794
2795     continue_port_dump:
2796
2797     cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
2798
2799     switch(cmd) {
2800     case PARTIAL:
2801     case DONE: /* DONE <handle> <label> <tape file> <err mess> */
2802         if(result_argc != 5) {
2803             error("error [dump to tape DONE result_argc != 5: %d]", result_argc);
2804             /*NOTREACHED*/
2805         }
2806
2807         if(failed == 1) goto tryagain;  /* dump didn't work */
2808         else if(failed == 2) goto failed_dumper;
2809
2810         free_serial(result_argv[2]);
2811
2812         dumpsize = (off_t)0;
2813         if (*result_argv[5] == '"') {
2814             /* String was quoted */
2815             rc = sscanf(result_argv[5],"\"[sec %lf kb " OFF_T_FMT " ",
2816                         &tapetime, (OFF_T_FMT_TYPE *)&dumpsize);
2817         } else {
2818             /* String was not quoted */
2819             rc = sscanf(result_argv[5],"[sec %lf kb " OFF_T_FMT " ",
2820                         &tapetime, (OFF_T_FMT_TYPE *)&dumpsize);
2821         }
2822         if (rc < 2) {
2823             error("error [malformed result: %d items matched in '%s']",
2824                   rc, result_argv[5]);
2825             /*NOTREACHED*/
2826         }
2827
2828         if(cmd == DONE) {
2829             /* every thing went fine */
2830             update_info_dumper(dp, origsize, dumpsize, dumptime);
2831             filenum = OFF_T_ATOI(result_argv[4]);
2832             update_info_taper(dp, result_argv[3], filenum, sched(dp)->level);
2833             /* note that update_info_dumper() must be run before
2834                update_info_taper(), since update_info_dumper overwrites
2835                tape information.  */
2836         }
2837
2838         break;
2839
2840     case TRYAGAIN: /* TRY-AGAIN <handle> <err mess> */
2841         tape_left = tape_length;
2842         current_tape++;
2843         if(dumper_tryagain == 0) {
2844             sched(dp)->attempted++;
2845             if(sched(dp)->attempted > failed)
2846                 failed = sched(dp)->attempted;
2847         }
2848     tryagain:
2849         if(failed <= 1)
2850             headqueue_disk(&runq, dp);
2851     failed_dumper:
2852         update_failed_dump_to_tape(dp);
2853         free_serial(result_argv[2]);
2854         break;
2855
2856     case SPLIT_CONTINUE:  /* SPLIT_CONTINUE <handle> <new_label> */
2857         if (result_argc != 3) {
2858             error("error [taper SPLIT_CONTINUE result_argc != 3: %d]", result_argc);
2859             /*NOTREACHED*/
2860         }
2861         fprintf(stderr, "driver: Got SPLIT_CONTINUE %s %s\n",
2862                 result_argv[2], result_argv[3]);
2863         goto continue_port_dump;
2864
2865     case SPLIT_NEEDNEXT:
2866         fprintf(stderr, "driver: Got SPLIT_NEEDNEXT %s %s\n", result_argv[2], result_argv[3]);
2867
2868         goto continue_port_dump;
2869
2870     case TAPE_ERROR: /* TAPE-ERROR <handle> <err mess> */
2871     case BOGUS:
2872     default:
2873         update_failed_dump_to_tape(dp);
2874         free_serial(result_argv[2]);
2875         failed = 2;     /* fatal problem */
2876         start_degraded_mode(&runq);
2877         break;
2878     }
2879
2880     /* reset statistics & return */
2881
2882     taper_busy = dumper->busy = 0;
2883     dp->host->inprogress -= 1;
2884     dp->inprogress = 0;
2885     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2886     amfree(qname);
2887
2888     return failed;
2889 }
2890
2891 static int
2892 queue_length(
2893     disklist_t  q)
2894 {
2895     disk_t *p;
2896     int len;
2897
2898     for(len = 0, p = q.head; p != NULL; len++, p = p->next)
2899         (void)len;      /* Quiet lint */
2900     return len;
2901 }
2902
2903 static void
2904 short_dump_state(void)
2905 {
2906     int i, nidle;
2907     char *wall_time;
2908
2909     wall_time = walltime_str(curclock());
2910
2911     printf("driver: state time %s ", wall_time);
2912     printf("free kps: %lu space: " OFF_T_FMT " taper: ",
2913            free_kps((interface_t *)0),
2914            (OFF_T_FMT_TYPE)free_space());
2915     if(degraded_mode) printf("DOWN");
2916     else if(!taper_busy) printf("idle");
2917     else printf("writing");
2918     nidle = 0;
2919     for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
2920     printf(" idle-dumpers: %d", nidle);
2921     printf(" qlen tapeq: %d", queue_length(tapeq));
2922     printf(" runq: %d", queue_length(runq));
2923     printf(" roomq: %d", queue_length(roomq));
2924     printf(" wakeup: %d", (int)sleep_time);
2925     printf(" driver-idle: %s\n", idle_strings[idle_reason]);
2926     interface_state(wall_time);
2927     holdingdisk_state(wall_time);
2928     fflush(stdout);
2929 }
2930
2931 #if 0
2932 static void
2933 dump_state(
2934     const char *str)
2935 {
2936     int i;
2937     disk_t *dp;
2938     char *qname;
2939
2940     printf("================\n");
2941     printf("driver state at time %s: %s\n", walltime_str(curclock()), str);
2942     printf("free kps: %lu, space: " OFF_T_FMT "\n",
2943            free_kps((interface_t *)0),
2944            (OFF_T_FMT_TYPE)free_space());
2945     if(degraded_mode) printf("taper: DOWN\n");
2946     else if(!taper_busy) printf("taper: idle\n");
2947     else printf("taper: writing %s:%s.%d est size " OFF_T_FMT "\n",
2948                 taper_disk->host->hostname, taper_disk->name,
2949                 sched(taper_disk)->level,
2950                 sched(taper_disk)->est_size);
2951     for(i = 0; i < inparallel; i++) {
2952         dp = dmptable[i].dp;
2953         if(!dmptable[i].busy)
2954           printf("%s: idle\n", dmptable[i].name);
2955         else
2956           qname = quote_string(dp->name);
2957           printf("%s: dumping %s:%s.%d est kps %d size " OFF_T_FMT " time %lu\n",
2958                 dmptable[i].name, dp->host->hostname, qname, sched(dp)->level,
2959                 sched(dp)->est_kps, sched(dp)->est_size, sched(dp)->est_time);
2960           amfree(qname);
2961     }
2962     dump_queue("TAPE", tapeq, 5, stdout);
2963     dump_queue("ROOM", roomq, 5, stdout);
2964     dump_queue("RUN ", runq, 5, stdout);
2965     printf("================\n");
2966     fflush(stdout);
2967 }
2968 #endif