git.gag.com Git - debian/amanda/blob - server-src/driver.c

   1 /*
   2  * Amanda, The Advanced Maryland Automatic Network Disk Archiver
   3  * Copyright (c) 1991-1998 University of Maryland at College Park
   4  * All Rights Reserved.
   5  *
   6  * Permission to use, copy, modify, distribute, and sell this software and its
   7  * documentation for any purpose is hereby granted without fee, provided that
   8  * the above copyright notice appear in all copies and that both that
   9  * copyright notice and this permission notice appear in supporting
  10  * documentation, and that the name of U.M. not be used in advertising or
  11  * publicity pertaining to distribution of the software without specific,
  12  * written prior permission.  U.M. makes no representations about the
  13  * suitability of this software for any purpose.  It is provided "as is"
  14  * without express or implied warranty.
  15  *
  16  * U.M. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL U.M.
  18  * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  19  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
  20  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
  21  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  22  *
  23  * Authors: the Amanda Development Team.  Its members are listed in a
  24  * file named AUTHORS, in the root directory of this distribution.
  25  */
  26 /*
  27  * $Id: driver.c,v 1.198 2006/08/24 01:57:16 paddy_s Exp $
  28  *
  29  * controlling process for the Amanda backup system
  30  */
  31
  32 /*
  33  * XXX possibly modify tape queue to be cognizant of how much room is left on
  34  *     tape.  Probably not effective though, should do this in planner.
  35  */
  36
  37 #include "amanda.h"
  38 #include "clock.h"
  39 #include "conffile.h"
  40 #include "diskfile.h"
  41 #include "event.h"
  42 #include "holding.h"
  43 #include "infofile.h"
  44 #include "logfile.h"
  45 #include "statfs.h"
  46 #include "version.h"
  47 #include "driverio.h"
  48 #include "server_util.h"
  49
  50 #define driver_debug(i,x) do {          \
  51         if ((i) <= debug_driver) {      \
  52             dbprintf(x);                \
  53         }                               \
  54 } while (0)
  55
  56 #define hold_debug(i,x) do {            \
  57         if ((i) <= debug_holding) {     \
  58             dbprintf(x);                \
  59         }                               \
  60 } while (0)
  61
  62 static disklist_t waitq, runq, tapeq, roomq;
  63 static int pending_aborts;
  64 static disk_t *taper_disk;
  65 static int degraded_mode;
  66 static off_t reserved_space;
  67 static off_t total_disksize;
  68 static char *dumper_program;
  69 static char *chunker_program;
  70 static int  inparallel;
  71 static int nodump = 0;
  72 static off_t tape_length = (off_t)0;
  73 static off_t tape_left = (off_t)0;
  74 static int current_tape = 1;
  75 static int conf_taperalgo;
  76 static int conf_runtapes;
  77 static time_t sleep_time;
  78 static int idle_reason;
  79 static char *driver_timestamp;
  80 static char *hd_driver_timestamp;
  81 static am_host_t *flushhost = NULL;
  82 static int need_degraded=0;
  83
  84 static event_handle_t *dumpers_ev_time = NULL;
  85 static event_handle_t *schedule_ev_read = NULL;
  86
  87 static int wait_children(int count);
  88 static void wait_for_children(void);
  89 static void allocate_bandwidth(interface_t *ip, unsigned long kps);
  90 static int assign_holdingdisk(assignedhd_t **holdp, disk_t *diskp);
  91 static void adjust_diskspace(disk_t *diskp, cmd_t cmd);
  92 static void delete_diskspace(disk_t *diskp);
  93 static assignedhd_t **build_diskspace(char *destname);
  94 static int client_constrained(disk_t *dp);
  95 static void deallocate_bandwidth(interface_t *ip, unsigned long kps);
  96 static void dump_schedule(disklist_t *qp, char *str);
  97 static int dump_to_tape(disk_t *dp);
  98 static assignedhd_t **find_diskspace(off_t size, int *cur_idle,
  99                                         assignedhd_t *preferred);
 100 static unsigned long free_kps(interface_t *ip);
 101 static off_t free_space(void);
 102 static void dumper_result(disk_t *dp);
 103 static void handle_dumper_result(void *);
 104 static void handle_chunker_result(void *);
 105 static void handle_dumpers_time(void *);
 106 static void handle_taper_result(void *);
 107 static void holdingdisk_state(char *time_str);
 108 static dumper_t *idle_dumper(void);
 109 static void interface_state(char *time_str);
 110 static int queue_length(disklist_t q);
 111 static disklist_t read_flush(void);
 112 static void read_schedule(void *cookie);
 113 static void short_dump_state(void);
 114 static void startaflush(void);
 115 static void start_degraded_mode(disklist_t *queuep);
 116 static void start_some_dumps(disklist_t *rq);
 117 static void continue_port_dumps(void);
 118 static void update_failed_dump_to_tape(disk_t *);
 119 #if 0
 120 static void dump_state(const char *str);
 121 #endif
 122 int main(int main_argc, char **main_argv);
 123
 124 static const char *idle_strings[] = {
 125 #define NOT_IDLE                0
 126     "not-idle",
 127 #define IDLE_NO_DUMPERS         1
 128     "no-dumpers",
 129 #define IDLE_START_WAIT         2
 130     "start-wait",
 131 #define IDLE_NO_HOLD            3
 132     "no-hold",
 133 #define IDLE_CLIENT_CONSTRAINED 4
 134     "client-constrained",
 135 #define IDLE_NO_DISKSPACE       5
 136     "no-diskspace",
 137 #define IDLE_TOO_LARGE          6
 138     "file-too-large",
 139 #define IDLE_NO_BANDWIDTH       7
 140     "no-bandwidth",
 141 #define IDLE_TAPER_WAIT         8
 142     "taper-wait",
 143 };
 144
 145 int
 146 main(
 147     int         main_argc,
 148     char **     main_argv)
 149 {
 150     disklist_t origq;
 151     disk_t *diskp;
 152     int dsk;
 153     dumper_t *dumper;
 154     char *newdir = NULL;
 155     generic_fs_stats_t fs;
 156     holdingdisk_t *hdp;
 157     unsigned long malloc_hist_1, malloc_size_1;
 158     unsigned long malloc_hist_2, malloc_size_2;
 159     unsigned long reserve = 100;
 160     char *conffile;
 161     char *conf_diskfile;
 162     cmd_t cmd;
 163     int result_argc;
 164     char *result_argv[MAX_ARGS+1];
 165     char *taper_program;
 166     char *conf_tapetype;
 167     tapetype_t *tape;
 168     char *line;
 169     int    new_argc,   my_argc;
 170     char **new_argv, **my_argv;
 171     char hostname[1025];
 172
 173     safe_fd(-1, 0);
 174
 175     setvbuf(stdout, (char *)NULL, (int)_IOLBF, 0);
 176     setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0);
 177
 178     set_pname("driver");
 179
 180     dbopen(DBG_SUBDIR_SERVER);
 181
 182     atexit(wait_for_children);
 183
 184     /* Don't die when child closes pipe */
 185     signal(SIGPIPE, SIG_IGN);
 186
 187     malloc_size_1 = malloc_inuse(&malloc_hist_1);
 188
 189     erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE);
 190     set_logerror(logerror);
 191
 192     startclock();
 193
 194     parse_conf(main_argc, main_argv, &new_argc, &new_argv);
 195     my_argc = new_argc;
 196     my_argv = new_argv;
 197
 198     printf("%s: pid %ld executable %s version %s\n",
 199            get_pname(), (long) getpid(), my_argv[0], version());
 200
 201     if (my_argc > 1) {
 202         config_name = stralloc(my_argv[1]);
 203         config_dir = vstralloc(CONFIG_DIR, "/", config_name, "/", NULL);
 204         if(my_argc > 2) {
 205             if(strncmp(my_argv[2], "nodump", 6) == 0) {
 206                 nodump = 1;
 207             }
 208         }
 209
 210     } else {
 211
 212         char my_cwd[STR_SIZE];
 213
 214         if (getcwd(my_cwd, SIZEOF(my_cwd)) == NULL) {
 215             error("cannot determine current working directory");
 216             /*NOTREACHED*/
 217         }
 218         config_dir = stralloc2(my_cwd, "/");
 219         if ((config_name = strrchr(my_cwd, '/')) != NULL) {
 220             config_name = stralloc(config_name + 1);
 221         }
 222     }
 223
 224     safe_cd();
 225
 226     conffile = stralloc2(config_dir, CONFFILE_NAME);
 227     if(read_conffile(conffile)) {
 228         error("errors processing config file \"%s\"", conffile);
 229         /*NOTREACHED*/
 230     }
 231     amfree(conffile);
 232
 233     dbrename(config_name, DBG_SUBDIR_SERVER);
 234
 235     report_bad_conf_arg();
 236
 237     amfree(driver_timestamp);
 238     /* read timestamp from stdin */
 239     while ((line = agets(stdin)) != NULL) {
 240         if (line[0] != '\0')
 241             break;
 242         amfree(line);
 243     }
 244     if ( line == NULL ) {
 245       error("Did not get DATE line from planner");
 246       /*NOTREACHED*/
 247     }
 248     driver_timestamp = alloc(15);
 249     strncpy(driver_timestamp, &line[5], 14);
 250     driver_timestamp[14] = '\0';
 251     amfree(line);
 252     log_add(L_START,"date %s", driver_timestamp);
 253
 254     gethostname(hostname, SIZEOF(hostname));
 255     log_add(L_STATS,"hostname %s", hostname);
 256
 257     /* check that we don't do many dump in a day and usetimestamps is off */
 258     if(strlen(driver_timestamp) == 8) {
 259         if (!nodump) {
 260             char *conf_logdir = getconf_str(CNF_LOGDIR);
 261             char *logfile    = vstralloc(conf_logdir, "/log.",
 262                                          driver_timestamp, ".0", NULL);
 263             char *oldlogfile = vstralloc(conf_logdir, "/oldlog/log.",
 264                                          driver_timestamp, ".0", NULL);
 265             if(access(logfile, F_OK) == 0 || access(oldlogfile, F_OK) == 0) {
 266                 log_add(L_WARNING, "WARNING: This is not the first amdump run today. Enable the usetimestamps option in the configuration file if you want to run amdump more than once per calendar day.");
 267             }
 268             amfree(oldlogfile);
 269             amfree(logfile);
 270         }
 271         hd_driver_timestamp = construct_timestamp(NULL);
 272     }
 273     else {
 274         hd_driver_timestamp = stralloc(driver_timestamp);
 275     }
 276
 277     taper_program = vstralloc(libexecdir, "/", "taper", versionsuffix(), NULL);
 278     dumper_program = vstralloc(libexecdir, "/", "dumper", versionsuffix(),
 279                                NULL);
 280     chunker_program = vstralloc(libexecdir, "/", "chunker", versionsuffix(),
 281                                NULL);
 282
 283     conf_taperalgo = getconf_taperalgo(CNF_TAPERALGO);
 284     conf_tapetype = getconf_str(CNF_TAPETYPE);
 285     conf_runtapes = getconf_int(CNF_RUNTAPES);
 286     tape = lookup_tapetype(conf_tapetype);
 287     tape_length = tapetype_get_length(tape);
 288     printf("driver: tape size " OFF_T_FMT "\n", (OFF_T_FMT_TYPE)tape_length);
 289
 290     /* start initializing: read in databases */
 291
 292     conf_diskfile = getconf_str(CNF_DISKFILE);
 293     if (*conf_diskfile == '/') {
 294         conf_diskfile = stralloc(conf_diskfile);
 295     } else {
 296         conf_diskfile = stralloc2(config_dir, conf_diskfile);
 297     }
 298     if (read_diskfile(conf_diskfile, &origq) < 0) {
 299         error("could not load disklist \"%s\"", conf_diskfile);
 300         /*NOTREACHED*/
 301     }
 302     amfree(conf_diskfile);
 303
 304     /* set up any configuration-dependent variables */
 305
 306     inparallel  = getconf_int(CNF_INPARALLEL);
 307
 308     reserve = (unsigned long)getconf_int(CNF_RESERVE);
 309
 310     total_disksize = (off_t)0;
 311     for(hdp = getconf_holdingdisks(), dsk = 0; hdp != NULL; hdp = hdp->next, dsk++) {
 312         hdp->up = (void *)alloc(SIZEOF(holdalloc_t));
 313         holdalloc(hdp)->allocated_dumpers = 0;
 314         holdalloc(hdp)->allocated_space = (off_t)0;
 315
 316         if(get_fs_stats(holdingdisk_get_diskdir(hdp), &fs) == -1
 317            || access(holdingdisk_get_diskdir(hdp), W_OK) == -1) {
 318             log_add(L_WARNING, "WARNING: ignoring holding disk %s: %s\n",
 319                     holdingdisk_get_diskdir(hdp), strerror(errno));
 320             hdp->disksize = 0L;
 321             continue;
 322         }
 323
 324         if(fs.avail != (off_t)-1) {
 325             if(hdp->disksize > (off_t)0) {
 326                 if(hdp->disksize > fs.avail) {
 327                     log_add(L_WARNING,
 328                             "WARNING: %s: " OFF_T_FMT " KB requested, "
 329                             "but only " OFF_T_FMT " KB available.",
 330                             holdingdisk_get_diskdir(hdp),
 331                             (OFF_T_FMT_TYPE)hdp->disksize,
 332                             (OFF_T_FMT_TYPE)fs.avail);
 333                             hdp->disksize = fs.avail;
 334                 }
 335             }
 336             else if((fs.avail + hdp->disksize) < (off_t)0) {
 337                 log_add(L_WARNING,
 338                         "WARNING: %s: not " OFF_T_FMT " KB free.",
 339                         holdingdisk_get_diskdir(hdp), -hdp->disksize);
 340                 hdp->disksize = (off_t)0;
 341                 continue;
 342             }
 343             else
 344                 hdp->disksize += fs.avail;
 345         }
 346
 347         printf("driver: adding holding disk %d dir %s size "
 348                 OFF_T_FMT " chunksize " OFF_T_FMT "\n",
 349                dsk, holdingdisk_get_diskdir(hdp),
 350                (OFF_T_FMT_TYPE)hdp->disksize,
 351                (OFF_T_FMT_TYPE)(holdingdisk_get_chunksize(hdp)));
 352
 353         newdir = newvstralloc(newdir,
 354                               holdingdisk_get_diskdir(hdp), "/", hd_driver_timestamp,
 355                               NULL);
 356         if(!mkholdingdir(newdir)) {
 357             hdp->disksize = (off_t)0;
 358         }
 359         total_disksize += hdp->disksize;
 360     }
 361
 362     reserved_space = total_disksize * (off_t)(reserve / 100);
 363
 364     printf("reserving " OFF_T_FMT " out of " OFF_T_FMT
 365            " for degraded-mode dumps\n",
 366            (OFF_T_FMT_TYPE)reserved_space, (OFF_T_FMT_TYPE)free_space());
 367
 368     amfree(newdir);
 369
 370     if(inparallel > MAX_DUMPERS) inparallel = MAX_DUMPERS;
 371
 372     /* taper takes a while to get going, so start it up right away */
 373
 374     init_driverio();
 375     if(conf_runtapes > 0) {
 376         startup_tape_process(taper_program);
 377         taper_cmd(START_TAPER, driver_timestamp, NULL, 0, NULL);
 378     }
 379
 380     /* fire up the dumpers now while we are waiting */
 381     if(!nodump) startup_dump_processes(dumper_program, inparallel, driver_timestamp);
 382
 383     /*
 384      * Read schedule from stdin.  Usually, this is a pipe from planner,
 385      * so the effect is that we wait here for the planner to
 386      * finish, but meanwhile the taper is rewinding the tape, reading
 387      * the label, checking it, writing a new label and all that jazz
 388      * in parallel with the planner.
 389      */
 390
 391     runq.head = NULL;
 392     runq.tail = NULL;
 393     waitq = origq;
 394     tapeq = read_flush();
 395
 396     roomq.head = roomq.tail = NULL;
 397
 398     log_add(L_STATS, "startup time %s", walltime_str(curclock()));
 399
 400     printf("driver: start time %s inparallel %d bandwidth %lu diskspace "
 401            OFF_T_FMT " ", walltime_str(curclock()), inparallel,
 402            free_kps((interface_t *)0), (OFF_T_FMT_TYPE)free_space());
 403     printf(" dir %s datestamp %s driver: drain-ends tapeq %s big-dumpers %s\n",
 404            "OBSOLETE", driver_timestamp, taperalgo2str(conf_taperalgo),
 405            getconf_str(CNF_DUMPORDER));
 406     fflush(stdout);
 407
 408     /* ok, planner is done, now lets see if the tape is ready */
 409
 410     if(conf_runtapes > 0) {
 411         cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
 412
 413         if(cmd != TAPER_OK) {
 414             /* no tape, go into degraded mode: dump to holding disk */
 415             need_degraded=1;
 416         }
 417     }
 418     else {
 419         need_degraded=1;
 420     }
 421
 422     tape_left = tape_length;
 423     taper_busy = 0;
 424     taper_disk = NULL;
 425     taper_ev_read = NULL;
 426     if(!need_degraded) startaflush();
 427
 428     if(!nodump)
 429         schedule_ev_read = event_register((event_id_t)0, EV_READFD, read_schedule, NULL);
 430
 431     short_dump_state();
 432     event_loop(0);
 433
 434     /* handle any remaining dumps by dumping directly to tape, if possible */
 435
 436     while(!empty(runq) && taper > 0) {
 437         diskp = dequeue_disk(&runq);
 438         if (diskp->to_holdingdisk == HOLD_REQUIRED) {
 439             log_add(L_FAIL, "%s %s %s %d [%s]",
 440                 diskp->host->hostname, diskp->name, sched(diskp)->datestamp,
 441                 sched(diskp)->level,
 442                 "can't dump required holdingdisk");
 443         }
 444         else if (!degraded_mode) {
 445             int rc = dump_to_tape(diskp);
 446             if(rc == 1)
 447                 log_add(L_INFO,
 448                         "%s %s %d [dump to tape failed, will try again]",
 449                         diskp->host->hostname,
 450                         diskp->name,
 451                         sched(diskp)->level);
 452             else if(rc == 2)
 453                 log_add(L_FAIL, "%s %s %s %d [dump to tape failed]",
 454                         diskp->host->hostname,
 455                         diskp->name,
 456                         sched(diskp)->datestamp,
 457                         sched(diskp)->level);
 458         }
 459         else
 460             log_add(L_FAIL, "%s %s %s %d [%s]",
 461                 diskp->host->hostname, diskp->name, sched(diskp)->datestamp,
 462                 sched(diskp)->level,
 463                 diskp->to_holdingdisk == HOLD_AUTO ?
 464                     "no more holding disk space" :
 465                     "can't dump no-hold disk in degraded mode");
 466     }
 467
 468     short_dump_state();                         /* for amstatus */
 469
 470     printf("driver: QUITTING time %s telling children to quit\n",
 471            walltime_str(curclock()));
 472     fflush(stdout);
 473
 474     if(!nodump) {
 475         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 476             if(dumper->fd >= 0)
 477                 dumper_cmd(dumper, QUIT, NULL);
 478         }
 479     }
 480
 481     if(taper >= 0) {
 482         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 483     }
 484
 485     /* wait for all to die */
 486     wait_children(600);
 487
 488     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
 489         cleanup_holdingdisk(holdingdisk_get_diskdir(hdp), 0);
 490         amfree(hdp->up);
 491     }
 492     amfree(newdir);
 493
 494     check_unfree_serial();
 495     printf("driver: FINISHED time %s\n", walltime_str(curclock()));
 496     fflush(stdout);
 497     log_add(L_FINISH,"date %s time %s", driver_timestamp, walltime_str(curclock()));
 498     amfree(driver_timestamp);
 499
 500     free_new_argv(new_argc, new_argv);
 501     amfree(dumper_program);
 502     amfree(taper_program);
 503     amfree(config_dir);
 504     amfree(config_name);
 505
 506     malloc_size_2 = malloc_inuse(&malloc_hist_2);
 507
 508     if(malloc_size_1 != malloc_size_2) {
 509         malloc_list(fileno(stderr), malloc_hist_1, malloc_hist_2);
 510     }
 511
 512     dbclose();
 513
 514     return 0;
 515 }
 516
 517 /* sleep up to count seconds, and wait for terminating child process */
 518 /* if sleep is negative, this function will not timeout              */
 519 /* exit once all child process are finished or the timout expired    */
 520 /* return 0 if no more children to wait                              */
 521 /* return 1 if some children are still alive                         */
 522 static int
 523 wait_children(int count)
 524 {
 525     pid_t     pid;
 526     amwait_t  retstat;
 527     char     *who;
 528     char     *what;
 529     int       code=0;
 530     dumper_t *dumper;
 531     int       wait_errno;
 532
 533     do {
 534         do {
 535             pid = waitpid((pid_t)-1, &retstat, WNOHANG);
 536             wait_errno = errno;
 537             if (pid > 0) {
 538                 what = NULL;
 539                 if (! WIFEXITED(retstat)) {
 540                     what = "signal";
 541                     code = WTERMSIG(retstat);
 542                 } else if (WEXITSTATUS(retstat) != 0) {
 543                     what = "code";
 544                     code = WEXITSTATUS(retstat);
 545                 }
 546                 who = NULL;
 547                 for (dumper = dmptable; dumper < dmptable + inparallel;
 548                      dumper++) {
 549                     if (pid == dumper->pid) {
 550                         who = stralloc(dumper->name);
 551                         dumper->pid = -1;
 552                         break;
 553                     }
 554                     if (dumper->chunker && pid == dumper->chunker->pid) {
 555                         who = stralloc(dumper->chunker->name);
 556                         dumper->chunker->pid = -1;
 557                         break;
 558                     }
 559                 }
 560                 if (who == NULL && pid == taper_pid) {
 561                     who = stralloc("taper");
 562                     taper_pid = -1;
 563                 }
 564                 if(what != NULL && who == NULL) {
 565                     who = stralloc("unknown");
 566                 }
 567                 if(who && what) {
 568                     log_add(L_WARNING, "%s pid %u exited with %s %d\n", who,
 569                             (unsigned)pid, what, code);
 570                     printf("driver: %s pid %u exited with %s %d\n", who,
 571                            (unsigned)pid, what, code);
 572                 }
 573                 amfree(who);
 574             }
 575         } while (pid > 0 || wait_errno == EINTR);
 576         if (errno != ECHILD)
 577             sleep(1);
 578         if (count > 0)
 579             count--;
 580     } while ((errno != ECHILD) && (count != 0));
 581     return (errno != ECHILD);
 582 }
 583
 584 static void
 585 kill_children(int signal)
 586 {
 587     dumper_t *dumper;
 588
 589     if(!nodump) {
 590         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 591             if (!dumper->down && dumper->pid > 1) {
 592                 printf("driver: sending signal %d to %s pid %u\n", signal,
 593                        dumper->name, (unsigned)dumper->pid);
 594                 if (kill(dumper->pid, signal) == -1 && errno == ESRCH) {
 595                     if (dumper->chunker)
 596                         dumper->chunker->pid = 0;
 597                 }
 598                 if (dumper->chunker && dumper->chunker->pid > 1) {
 599                     printf("driver: sending signal %d to %s pid %u\n", signal,
 600                            dumper->chunker->name,
 601                            (unsigned)dumper->chunker->pid);
 602                     if (kill(dumper->chunker->pid, signal) == -1 &&
 603                         errno == ESRCH)
 604                         dumper->chunker->pid = 0;
 605                 }
 606             }
 607         }
 608     }
 609
 610     if(taper_pid > 1)
 611         printf("driver: sending signal %d to %s pid %u\n", signal,
 612                "taper", (unsigned)taper_pid);
 613         if (kill(taper_pid, signal) == -1 && errno == ESRCH)
 614             taper_pid = 0;
 615 }
 616
 617 static void
 618 wait_for_children(void)
 619 {
 620     dumper_t *dumper;
 621
 622     if(!nodump) {
 623         for(dumper = dmptable; dumper < dmptable + inparallel; dumper++) {
 624             if (dumper->pid > 1 && dumper->fd >= 0) {
 625                 dumper_cmd(dumper, QUIT, NULL);
 626                 if (dumper->chunker && dumper->chunker->pid > 1 &&
 627                     dumper->chunker->fd >= 0)
 628                     chunker_cmd(dumper->chunker, QUIT, NULL);
 629             }
 630         }
 631     }
 632
 633     if(taper_pid > 1 && taper > 0) {
 634         taper_cmd(QUIT, NULL, NULL, 0, NULL);
 635     }
 636
 637     if(wait_children(60) == 0)
 638         return;
 639
 640     kill_children(SIGHUP);
 641     if(wait_children(60) == 0)
 642         return;
 643
 644     kill_children(SIGKILL);
 645     if(wait_children(-1) == 0)
 646         return;
 647
 648 }
 649
 650 static void
 651 startaflush(void)
 652 {
 653     disk_t *dp = NULL;
 654     disk_t *fit = NULL;
 655     char *datestamp;
 656     int extra_tapes = 0;
 657     char *qname;
 658
 659     if(!degraded_mode && !taper_busy && !empty(tapeq)) {
 660
 661         datestamp = sched(tapeq.head)->datestamp;
 662         switch(conf_taperalgo) {
 663         case ALGO_FIRST:
 664                 dp = dequeue_disk(&tapeq);
 665                 break;
 666         case ALGO_FIRSTFIT:
 667                 fit = tapeq.head;
 668                 while (fit != NULL) {
 669                     extra_tapes = (fit->tape_splitsize > (off_t)0) ?
 670                                         conf_runtapes - current_tape : 0;
 671                     if(sched(fit)->act_size <= (tape_left +
 672                              tape_length * (off_t)extra_tapes) &&
 673                              strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 674                         dp = fit;
 675                         fit = NULL;
 676                     }
 677                     else {
 678                         fit = fit->next;
 679                     }
 680                 }
 681                 if(dp) remove_disk(&tapeq, dp);
 682                 break;
 683         case ALGO_LARGEST:
 684                 fit = dp = tapeq.head;
 685                 while (fit != NULL) {
 686                     if(sched(fit)->act_size > sched(dp)->act_size &&
 687                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 688                         dp = fit;
 689                     }
 690                     fit = fit->next;
 691                 }
 692                 if(dp) remove_disk(&tapeq, dp);
 693                 break;
 694         case ALGO_LARGESTFIT:
 695                 fit = tapeq.head;
 696                 while (fit != NULL) {
 697                     extra_tapes = (fit->tape_splitsize > (off_t)0) ?
 698                                         conf_runtapes - current_tape : 0;
 699                     if(sched(fit)->act_size <=
 700                        (tape_left + tape_length * (off_t)extra_tapes) &&
 701                        (!dp || sched(fit)->act_size > sched(dp)->act_size) &&
 702                        strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 703                         dp = fit;
 704                     }
 705                     fit = fit->next;
 706                 }
 707                 if(dp) remove_disk(&tapeq, dp);
 708                 break;
 709         case ALGO_SMALLEST:
 710                 break;
 711         case ALGO_LAST:
 712                 dp = tapeq.tail;
 713                 remove_disk(&tapeq, dp);
 714                 break;
 715         }
 716         if(!dp) { /* ALGO_SMALLEST, or default if nothing fit. */
 717             if(conf_taperalgo != ALGO_SMALLEST)  {
 718                 fprintf(stderr,
 719                    "driver: startaflush: Using SMALLEST because nothing fit\n");
 720             }
 721             fit = dp = tapeq.head;
 722             while (fit != NULL) {
 723                 if(sched(fit)->act_size < sched(dp)->act_size &&
 724                    strcmp(sched(fit)->datestamp, datestamp) <= 0) {
 725                     dp = fit;
 726                 }
 727                 fit = fit->next;
 728             }
 729             if(dp) remove_disk(&tapeq, dp);
 730         }
 731         if(taper_ev_read == NULL) {
 732             taper_ev_read = event_register((event_id_t)taper, EV_READFD,
 733                                            handle_taper_result, NULL);
 734         }
 735         if (dp) {
 736             taper_disk = dp;
 737             taper_busy = 1;
 738             qname = quote_string(dp->name);
 739             taper_cmd(FILE_WRITE, dp, sched(dp)->destname, sched(dp)->level,
 740                       sched(dp)->datestamp);
 741             fprintf(stderr,"driver: startaflush: %s %s %s "
 742                     OFF_T_FMT " " OFF_T_FMT "\n",
 743                     taperalgo2str(conf_taperalgo), dp->host->hostname, qname,
 744                     (OFF_T_FMT_TYPE)sched(taper_disk)->act_size,
 745                     (OFF_T_FMT_TYPE)tape_left);
 746             if(sched(dp)->act_size <= tape_left)
 747                 tape_left -= sched(dp)->act_size;
 748             else
 749                 tape_left = (off_t)0;
 750             amfree(qname);
 751         } else {
 752             error("FATAL: Taper marked busy and no work found.");
 753             /*NOTREACHED*/
 754         }
 755     } else if(!taper_busy && taper_ev_read != NULL) {
 756         event_release(taper_ev_read);
 757         taper_ev_read = NULL;
 758     }
 759 }
 760
 761
 762 static int
 763 client_constrained(
 764     disk_t *    dp)
 765 {
 766     disk_t *dp2;
 767
 768     /* first, check if host is too busy */
 769
 770     if(dp->host->inprogress >= dp->host->maxdumps) {
 771         return 1;
 772     }
 773
 774     /* next, check conflict with other dumps on same spindle */
 775
 776     if(dp->spindle == -1) {     /* but spindle -1 never conflicts by def. */
 777         return 0;
 778     }
 779
 780     for(dp2 = dp->host->disks; dp2 != NULL; dp2 = dp2->hostnext)
 781         if(dp2->inprogress && dp2->spindle == dp->spindle) {
 782             return 1;
 783         }
 784
 785     return 0;
 786 }
 787
 788 static void
 789 start_some_dumps(
 790     disklist_t *        rq)
 791 {
 792     int cur_idle;
 793     disk_t *diskp, *delayed_diskp, *diskp_accept;
 794     assignedhd_t **holdp=NULL, **holdp_accept;
 795     const time_t now = time(NULL);
 796     cmd_t cmd;
 797     int result_argc;
 798     char *result_argv[MAX_ARGS+1];
 799     chunker_t *chunker;
 800     dumper_t *dumper;
 801     char dumptype;
 802     char *dumporder;
 803
 804     idle_reason = IDLE_NO_DUMPERS;
 805     sleep_time = 0;
 806
 807     if(dumpers_ev_time != NULL) {
 808         event_release(dumpers_ev_time);
 809         dumpers_ev_time = NULL;
 810     }
 811
 812     for (dumper = dmptable; dumper < dmptable+inparallel; dumper++) {
 813
 814         if( dumper->busy || dumper->down) {
 815             continue;
 816         }
 817
 818         if (dumper->ev_read != NULL) {
 819             event_release(dumper->ev_read);
 820             dumper->ev_read = NULL;
 821         }
 822
 823         /*
 824          * A potential problem with starting from the bottom of the dump time
 825          * distribution is that a slave host will have both one of the shortest
 826          * and one of the longest disks, so starting its shortest disk first will
 827          * tie up the host and eliminate its longest disk from consideration the
 828          * first pass through.  This could cause a big delay in starting that long
 829          * disk, which could drag out the whole night's dumps.
 830          *
 831          * While starting from the top of the dump time distribution solves the
 832          * above problem, this turns out to be a bad idea, because the big dumps
 833          * will almost certainly pack the holding disk completely, leaving no
 834          * room for even one small dump to start.  This ends up shutting out the
 835          * small-end dumpers completely (they stay idle).
 836          *
 837          * The introduction of multiple simultaneous dumps to one host alleviates
 838          * the biggest&smallest dumps problem: both can be started at the
 839          * beginning.
 840          */
 841
 842         diskp_accept = NULL;
 843         holdp_accept = NULL;
 844         delayed_diskp = NULL;
 845
 846         cur_idle = NOT_IDLE;
 847
 848         dumporder = getconf_str(CNF_DUMPORDER);
 849         if(strlen(dumporder) > (size_t)(dumper-dmptable)) {
 850             dumptype = dumporder[dumper-dmptable];
 851         }
 852         else {
 853             if(dumper-dmptable < 3)
 854                 dumptype = 't';
 855             else
 856                 dumptype = 'T';
 857         }
 858
 859         for(diskp = rq->head; diskp != NULL; diskp = diskp->next) {
 860             assert(diskp->host != NULL && sched(diskp) != NULL);
 861
 862             if (diskp->host->start_t > now) {
 863                 cur_idle = max(cur_idle, IDLE_START_WAIT);
 864                 if (delayed_diskp == NULL || sleep_time > diskp->host->start_t) {
 865                     delayed_diskp = diskp;
 866                     sleep_time = diskp->host->start_t;
 867                 }
 868             } else if(diskp->start_t > now) {
 869                 cur_idle = max(cur_idle, IDLE_START_WAIT);
 870                 if (delayed_diskp == NULL || sleep_time > diskp->start_t) {
 871                     delayed_diskp = diskp;
 872                     sleep_time = diskp->start_t;
 873                 }
 874             } else if (diskp->host->netif->curusage > 0 &&
 875                        sched(diskp)->est_kps > free_kps(diskp->host->netif)) {
 876                 cur_idle = max(cur_idle, IDLE_NO_BANDWIDTH);
 877             } else if(sched(diskp)->no_space) {
 878                 cur_idle = max(cur_idle, IDLE_NO_DISKSPACE);
 879             } else if (diskp->to_holdingdisk == HOLD_NEVER) {
 880                 cur_idle = max(cur_idle, IDLE_NO_HOLD);
 881             } else if ((holdp =
 882                 find_diskspace(sched(diskp)->est_size, &cur_idle, NULL)) == NULL) {
 883                 cur_idle = max(cur_idle, IDLE_NO_DISKSPACE);
 884             } else if (client_constrained(diskp)) {
 885                 free_assignedhd(holdp);
 886                 cur_idle = max(cur_idle, IDLE_CLIENT_CONSTRAINED);
 887             } else {
 888
 889                 /* disk fits, dump it */
 890                 int accept = !diskp_accept;
 891                 if(!accept) {
 892                     switch(dumptype) {
 893                       case 's': accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size);
 894                                 break;
 895                       case 'S': accept = (sched(diskp)->est_size > sched(diskp_accept)->est_size);
 896                                 break;
 897                       case 't': accept = (sched(diskp)->est_time < sched(diskp_accept)->est_time);
 898                                 break;
 899                       case 'T': accept = (sched(diskp)->est_time > sched(diskp_accept)->est_time);
 900                                 break;
 901                       case 'b': accept = (sched(diskp)->est_kps < sched(diskp_accept)->est_kps);
 902                                 break;
 903                       case 'B': accept = (sched(diskp)->est_kps > sched(diskp_accept)->est_kps);
 904                                 break;
 905                       default:  log_add(L_WARNING, "Unknown dumporder character \'%c\', using 's'.\n",
 906                                         dumptype);
 907                                 accept = (sched(diskp)->est_size < sched(diskp_accept)->est_size);
 908                                 break;
 909                     }
 910                 }
 911                 if(accept) {
 912                     if( !diskp_accept || !degraded_mode || diskp->priority >= diskp_accept->priority) {
 913                         if(holdp_accept) free_assignedhd(holdp_accept);
 914                         diskp_accept = diskp;
 915                         holdp_accept = holdp;
 916                     }
 917                     else {
 918                         free_assignedhd(holdp);
 919                     }
 920                 }
 921                 else {
 922                     free_assignedhd(holdp);
 923                 }
 924             }
 925         }
 926
 927         diskp = diskp_accept;
 928         holdp = holdp_accept;
 929
 930         idle_reason = max(idle_reason, cur_idle);
 931
 932         /*
 933          * If we have no disk at this point, and there are disks that
 934          * are delayed, then schedule a time event to call this dumper
 935          * with the disk with the shortest delay.
 936          */
 937         if (diskp == NULL && delayed_diskp != NULL) {
 938             assert(sleep_time > now);
 939             sleep_time -= now;
 940             dumpers_ev_time = event_register((event_id_t)sleep_time, EV_TIME,
 941                 handle_dumpers_time, &runq);
 942             return;
 943         } else if (diskp != NULL) {
 944             sched(diskp)->act_size = (off_t)0;
 945             allocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
 946             sched(diskp)->activehd = assign_holdingdisk(holdp, diskp);
 947             amfree(holdp);
 948             sched(diskp)->destname = newstralloc(sched(diskp)->destname,
 949                                                  sched(diskp)->holdp[0]->destname);
 950             diskp->host->inprogress++;  /* host is now busy */
 951             diskp->inprogress = 1;
 952             sched(diskp)->dumper = dumper;
 953             sched(diskp)->timestamp = now;
 954
 955             dumper->busy = 1;           /* dumper is now busy */
 956             dumper->dp = diskp;         /* link disk to dumper */
 957             remove_disk(rq, diskp);             /* take it off the run queue */
 958
 959             sched(diskp)->origsize = (off_t)-1;
 960             sched(diskp)->dumpsize = (off_t)-1;
 961             sched(diskp)->dumptime = (time_t)0;
 962             sched(diskp)->tapetime = (time_t)0;
 963             chunker = dumper->chunker;
 964             chunker->result = LAST_TOK;
 965             dumper->result = LAST_TOK;
 966             startup_chunk_process(chunker,chunker_program);
 967             chunker_cmd(chunker, START, (void *)driver_timestamp);
 968             chunker->dumper = dumper;
 969             chunker_cmd(chunker, PORT_WRITE, diskp);
 970             cmd = getresult(chunker->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
 971             if(cmd != PORT) {
 972                 assignedhd_t **h=NULL;
 973                 int activehd;
 974
 975                 printf("driver: did not get PORT from %s for %s:%s\n",
 976                        chunker->name, diskp->host->hostname, diskp->name);
 977                 fflush(stdout);
 978
 979                 deallocate_bandwidth(diskp->host->netif, sched(diskp)->est_kps);
 980                 h = sched(diskp)->holdp;
 981                 activehd = sched(diskp)->activehd;
 982                 h[activehd]->used = 0;
 983                 holdalloc(h[activehd]->disk)->allocated_dumpers--;
 984                 adjust_diskspace(diskp, DONE);
 985                 delete_diskspace(diskp);
 986                 diskp->host->inprogress--;
 987                 diskp->inprogress = 0;
 988                 sched(diskp)->dumper = NULL;
 989                 dumper->busy = 0;
 990                 dumper->dp = NULL;
 991                 sched(diskp)->attempted++;
 992                 free_serial_dp(diskp);
 993                 if(sched(diskp)->attempted < 2)
 994                     enqueue_disk(rq, diskp);
 995             }
 996             else {
 997                 dumper->ev_read = event_register((event_id_t)dumper->fd, EV_READFD,
 998                                                  handle_dumper_result, dumper);
 999                 chunker->ev_read = event_register((event_id_t)chunker->fd, EV_READFD,
1000                                                    handle_chunker_result, chunker);
1001                 dumper->output_port = atoi(result_argv[2]);
1002
1003                 dumper_cmd(dumper, PORT_DUMP, diskp);
1004             }
1005             diskp->host->start_t = now + 15;
1006         }
1007     }
1008 }
1009
1010 /*
1011  * This gets called when a dumper is delayed for some reason.  It may
1012  * be because a disk has a delayed start, or amanda is constrained
1013  * by network or disk limits.
1014  */
1015
1016 static void
1017 handle_dumpers_time(
1018     void *      cookie)
1019 {
1020     disklist_t *runq = cookie;
1021     event_release(dumpers_ev_time);
1022     dumpers_ev_time = NULL;
1023     start_some_dumps(runq);
1024 }
1025
1026 static void
1027 dump_schedule(
1028     disklist_t *qp,
1029     char *      str)
1030 {
1031     disk_t *dp;
1032     char *qname;
1033
1034     printf("dump of driver schedule %s:\n--------\n", str);
1035
1036     for(dp = qp->head; dp != NULL; dp = dp->next) {
1037         qname = quote_string(dp->name);
1038         printf("  %-20s %-25s lv %d t %5lu s " OFF_T_FMT " p %d\n",
1039                dp->host->hostname, qname, sched(dp)->level,
1040                sched(dp)->est_time,
1041                (OFF_T_FMT_TYPE)sched(dp)->est_size, sched(dp)->priority);
1042         amfree(qname);
1043     }
1044     printf("--------\n");
1045 }
1046
1047 static void
1048 start_degraded_mode(
1049     /*@keep@*/ disklist_t *queuep)
1050 {
1051     disk_t *dp;
1052     disklist_t newq;
1053     off_t est_full_size;
1054     char *qname;
1055
1056     if (taper_ev_read != NULL) {
1057         event_release(taper_ev_read);
1058         taper_ev_read = NULL;
1059     }
1060
1061     newq.head = newq.tail = 0;
1062
1063     dump_schedule(queuep, "before start degraded mode");
1064
1065     est_full_size = (off_t)0;
1066     while(!empty(*queuep)) {
1067         dp = dequeue_disk(queuep);
1068
1069         qname = quote_string(dp->name);
1070         if(sched(dp)->level != 0)
1071             /* go ahead and do the disk as-is */
1072             enqueue_disk(&newq, dp);
1073         else {
1074             if (reserved_space + est_full_size + sched(dp)->est_size
1075                 <= total_disksize) {
1076                 enqueue_disk(&newq, dp);
1077                 est_full_size += sched(dp)->est_size;
1078             }
1079             else if(sched(dp)->degr_level != -1) {
1080                 sched(dp)->level = sched(dp)->degr_level;
1081                 sched(dp)->dumpdate = sched(dp)->degr_dumpdate;
1082                 sched(dp)->est_nsize = sched(dp)->degr_nsize;
1083                 sched(dp)->est_csize = sched(dp)->degr_csize;
1084                 sched(dp)->est_time = sched(dp)->degr_time;
1085                 sched(dp)->est_kps  = sched(dp)->degr_kps;
1086                 enqueue_disk(&newq, dp);
1087             }
1088             else {
1089                 log_add(L_FAIL,"%s %s %s %d [can't switch to incremental dump]",
1090                         dp->host->hostname, qname, sched(dp)->datestamp,
1091                         sched(dp)->level);
1092             }
1093         }
1094         amfree(qname);
1095     }
1096
1097     /*@i@*/ *queuep = newq;
1098     degraded_mode = 1;
1099
1100     dump_schedule(queuep, "after start degraded mode");
1101 }
1102
1103
1104 static void
1105 continue_port_dumps(void)
1106 {
1107     disk_t *dp, *ndp;
1108     assignedhd_t **h;
1109     int active_dumpers=0, busy_dumpers=0, i;
1110     dumper_t *dumper;
1111
1112     /* First we try to grant diskspace to some dumps waiting for it. */
1113     for( dp = roomq.head; dp; dp = ndp ) {
1114         ndp = dp->next;
1115         /* find last holdingdisk used by this dump */
1116         for( i = 0, h = sched(dp)->holdp; h[i+1]; i++ ) {
1117             (void)h; /* Quiet lint */
1118         }
1119         /* find more space */
1120         h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1121                             &active_dumpers, h[i] );
1122         if( h ) {
1123             for(dumper = dmptable; dumper < dmptable + inparallel &&
1124                                    dumper->dp != dp; dumper++) {
1125                 (void)dp; /* Quiet lint */
1126             }
1127             assert( dumper < dmptable + inparallel );
1128             sched(dp)->activehd = assign_holdingdisk( h, dp );
1129             chunker_cmd( dumper->chunker, CONTINUE, dp );
1130             amfree(h);
1131             remove_disk( &roomq, dp );
1132         }
1133     }
1134
1135     /* So for some disks there is less holding diskspace available than
1136      * was asked for. Possible reasons are
1137      * a) diskspace has been allocated for other dumps which are
1138      *    still running or already being written to tape
1139      * b) all other dumps have been suspended due to lack of diskspace
1140      * c) this dump doesn't fit on all the holding disks
1141      * Case a) is not a problem. We just wait for the diskspace to
1142      * be freed by moving the current disk to a queue.
1143      * If case b) occurs, we have a deadlock situation. We select
1144      * a dump from the queue to be aborted and abort it. It will
1145      * be retried later dumping to disk.
1146      * If case c) is detected, the dump is aborted. Next time
1147      * it will be dumped directly to tape. Actually, case c is a special
1148      * manifestation of case b) where only one dumper is busy.
1149      */
1150     for(dp=NULL, dumper = dmptable; dumper < (dmptable+inparallel); dumper++) {
1151         if( dumper->busy ) {
1152             busy_dumpers++;
1153             if( !find_disk(&roomq, dumper->dp) ) {
1154                 active_dumpers++;
1155             } else if( !dp ||
1156                        sched(dp)->est_size > sched(dumper->dp)->est_size ) {
1157                 dp = dumper->dp;
1158             }
1159         }
1160     }
1161     if((dp != NULL) && (active_dumpers == 0) && (busy_dumpers > 0) &&
1162         ((!taper_busy && empty(tapeq)) || degraded_mode) &&
1163         pending_aborts == 0 ) { /* not case a */
1164         if( busy_dumpers == 1 ) { /* case c */
1165             sched(dp)->no_space = 1;
1166         }
1167         /* case b */
1168         /* At this time, dp points to the dump with the smallest est_size.
1169          * We abort that dump, hopefully not wasting too much time retrying it.
1170          */
1171         remove_disk( &roomq, dp );
1172         chunker_cmd( sched(dp)->dumper->chunker, ABORT, NULL);
1173         dumper_cmd( sched(dp)->dumper, ABORT, NULL );
1174         pending_aborts++;
1175     }
1176 }
1177
1178
1179 static void
1180 handle_taper_result(
1181     void *      cookie)
1182 {
1183     disk_t *dp;
1184     off_t filenum;
1185     cmd_t cmd;
1186     int result_argc;
1187     char *result_argv[MAX_ARGS+1];
1188     int avail_tapes = 0;
1189
1190     (void)cookie;       /* Quiet unused parameter warning */
1191
1192     assert(cookie == NULL);
1193
1194     do {
1195
1196         short_dump_state();
1197
1198         cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
1199
1200         switch(cmd) {
1201
1202         case PARTIAL:
1203         case DONE:      /* DONE <handle> <label> <tape file> <err mess> */
1204             if(result_argc != 5) {
1205                 error("error: [taper DONE result_argc != 5: %d", result_argc);
1206                 /*NOTREACHED*/
1207             }
1208
1209             dp = serial2disk(result_argv[2]);
1210             free_serial(result_argv[2]);
1211
1212             filenum = OFF_T_ATOI(result_argv[4]);
1213             if(cmd == DONE) {
1214                 update_info_taper(dp, result_argv[3], filenum,
1215                                   sched(dp)->level);
1216             }
1217
1218             delete_diskspace(dp);
1219
1220             printf("driver: finished-cmd time %s taper wrote %s:%s\n",
1221                    walltime_str(curclock()), dp->host->hostname, dp->name);
1222             fflush(stdout);
1223
1224             amfree(sched(dp)->destname);
1225             amfree(sched(dp)->dumpdate);
1226             amfree(sched(dp)->degr_dumpdate);
1227             amfree(sched(dp)->datestamp);
1228             amfree(dp->up);
1229
1230             taper_busy = 0;
1231             taper_disk = NULL;
1232             startaflush();
1233
1234             /* continue with those dumps waiting for diskspace */
1235             continue_port_dumps();
1236             break;
1237
1238         case TRYAGAIN:  /* TRY-AGAIN <handle> <err mess> */
1239             if (result_argc < 2) {
1240                 error("error [taper TRYAGAIN result_argc < 2: %d]",
1241                       result_argc);
1242                 /*NOTREACHED*/
1243             }
1244             dp = serial2disk(result_argv[2]);
1245             free_serial(result_argv[2]);
1246             printf("driver: taper-tryagain time %s disk %s:%s\n",
1247                    walltime_str(curclock()), dp->host->hostname, dp->name);
1248             fflush(stdout);
1249
1250             /* See how many tapes we have left, but we alwyays
1251                retry once (why?) */
1252             current_tape++;
1253             if(dp->tape_splitsize > (off_t)0)
1254                 avail_tapes = conf_runtapes - current_tape;
1255             else
1256                 avail_tapes = 0;
1257
1258             if(sched(dp)->attempted > avail_tapes) {
1259                 log_add(L_FAIL, "%s %s %s %d [too many taper retries]",
1260                         dp->host->hostname, dp->name, sched(dp)->datestamp,
1261                         sched(dp)->level);
1262                 printf("driver: taper failed %s %s %s, too many taper retry\n",
1263                        result_argv[2], dp->host->hostname, dp->name);
1264             }
1265             else {
1266                 /* Re-insert into taper queue. */
1267                 sched(dp)->attempted++;
1268                 headqueue_disk(&tapeq, dp);
1269             }
1270
1271             tape_left = tape_length;
1272
1273             /* run next thing from queue */
1274
1275             taper_busy = 0;
1276             taper_disk = NULL;
1277             startaflush();
1278             continue_port_dumps();
1279             break;
1280
1281         case SPLIT_CONTINUE:  /* SPLIT_CONTINUE <handle> <new_label> */
1282             if (result_argc != 3) {
1283                 error("error [taper SPLIT_CONTINUE result_argc != 3: %d]",
1284                       result_argc);
1285                 /*NOTREACHED*/
1286             }
1287
1288             break;
1289         case SPLIT_NEEDNEXT:  /* SPLIT-NEEDNEXT <handle> <kb written> */
1290             if (result_argc != 3) {
1291                 error("error [taper SPLIT_NEEDNEXT result_argc != 3: %d]",
1292                       result_argc);
1293                 /*NOTREACHED*/
1294             }
1295
1296             /* Update our tape counter and reset tape_left */
1297             current_tape++;
1298             tape_left = tape_length;
1299
1300             /* Reduce the size of the dump by amount written and reduce
1301                tape_left by the amount left over */
1302             dp = serial2disk(result_argv[2]);
1303             sched(dp)->act_size -= OFF_T_ATOI(result_argv[3]);
1304             if (sched(dp)->act_size < tape_left)
1305                 tape_left -= sched(dp)->act_size;
1306             else
1307                 tape_length = 0;
1308
1309             break;
1310
1311         case TAPE_ERROR: /* TAPE-ERROR <handle> <err mess> */
1312             dp = serial2disk(result_argv[2]);
1313             free_serial(result_argv[2]);
1314             printf("driver: finished-cmd time %s taper wrote %s:%s\n",
1315                    walltime_str(curclock()), dp->host->hostname, dp->name);
1316             fflush(stdout);
1317             log_add(L_WARNING, "Taper  error: %s", result_argv[3]);
1318             /*FALLTHROUGH*/
1319
1320         case BOGUS:
1321             if (cmd == BOGUS) {
1322                 log_add(L_WARNING, "Taper protocol error");
1323             }
1324             /*
1325              * Since we received a taper error, we can't send anything more
1326              * to the taper.  Go into degraded mode to try to get everthing
1327              * onto disk.  Later, these dumps can be flushed to a new tape.
1328              * The tape queue is zapped so that it appears empty in future
1329              * checks. If there are dumps waiting for diskspace to be freed,
1330              * cancel one.
1331              */
1332             if(!nodump) {
1333                 log_add(L_WARNING,
1334                         "going into degraded mode because of taper component error.");
1335                 start_degraded_mode(&runq);
1336             }
1337             tapeq.head = tapeq.tail = NULL;
1338             taper_busy = 0;
1339             taper_disk = NULL;
1340             if(taper_ev_read != NULL) {
1341                 event_release(taper_ev_read);
1342                 taper_ev_read = NULL;
1343             }
1344             if(cmd != TAPE_ERROR) aclose(taper);
1345             continue_port_dumps();
1346             break;
1347
1348         default:
1349             error("driver received unexpected token (%s) from taper",
1350                   cmdstr[cmd]);
1351             /*NOTREACHED*/
1352         }
1353         /*
1354          * Wakeup any dumpers that are sleeping because of network
1355          * or disk constraints.
1356          */
1357         start_some_dumps(&runq);
1358
1359     } while(areads_dataready(taper));
1360 }
1361
1362 static dumper_t *
1363 idle_dumper(void)
1364 {
1365     dumper_t *dumper;
1366
1367     for(dumper = dmptable; dumper < dmptable+inparallel; dumper++)
1368         if(!dumper->busy && !dumper->down) return dumper;
1369
1370     return NULL;
1371 }
1372
1373 static void
1374 dumper_result(
1375     disk_t *    dp)
1376 {
1377     dumper_t *dumper;
1378     chunker_t *chunker;
1379     assignedhd_t **h=NULL;
1380     int activehd, i;
1381     off_t dummy;
1382     off_t size;
1383     int is_partial;
1384
1385     dumper = sched(dp)->dumper;
1386     chunker = dumper->chunker;
1387
1388     free_serial_dp(dp);
1389
1390     h = sched(dp)->holdp;
1391     activehd = sched(dp)->activehd;
1392
1393     if(dumper->result == DONE && chunker->result == DONE) {
1394         update_info_dumper(dp, sched(dp)->origsize,
1395                            sched(dp)->dumpsize, sched(dp)->dumptime);
1396         log_add(L_STATS, "estimate %s %s %s %d [sec %ld nkb " OFF_T_FMT
1397                 " ckb " OFF_T_FMT " kps %lu]",
1398                 dp->host->hostname, dp->name, sched(dp)->datestamp,
1399                 sched(dp)->level,
1400                 sched(dp)->est_time, (OFF_T_FMT_TYPE)sched(dp)->est_nsize,
1401                 (OFF_T_FMT_TYPE)sched(dp)->est_csize,
1402                 sched(dp)->est_kps);
1403     }
1404
1405     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
1406
1407     is_partial = dumper->result != DONE || chunker->result != DONE;
1408     rename_tmp_holding(sched(dp)->destname, !is_partial);
1409
1410     dummy = (off_t)0;
1411     for( i = 0, h = sched(dp)->holdp; i < activehd; i++ ) {
1412         dummy += h[i]->used;
1413     }
1414
1415     size = holding_file_size(sched(dp)->destname, 0);
1416     h[activehd]->used = size - dummy;
1417     holdalloc(h[activehd]->disk)->allocated_dumpers--;
1418     adjust_diskspace(dp, DONE);
1419
1420     sched(dp)->attempted += 1;
1421
1422     if((dumper->result != DONE || chunker->result != DONE) &&
1423        sched(dp)->attempted <= 1) {
1424         delete_diskspace(dp);
1425         enqueue_disk(&runq, dp);
1426     }
1427     else if(size > (off_t)DISK_BLOCK_KB) {
1428         sched(dp)->attempted = 0;
1429         enqueue_disk(&tapeq, dp);
1430         startaflush();
1431     }
1432     else {
1433         delete_diskspace(dp);
1434     }
1435
1436     dumper->busy = 0;
1437     dp->host->inprogress -= 1;
1438     dp->inprogress = 0;
1439
1440     waitpid(chunker->pid, NULL, 0 );
1441     aclose(chunker->fd);
1442     chunker->fd = -1;
1443     chunker->down = 1;
1444
1445     dp = NULL;
1446     if (chunker->result == ABORT_FINISHED)
1447         pending_aborts--;
1448     continue_port_dumps();
1449     /*
1450      * Wakeup any dumpers that are sleeping because of network
1451      * or disk constraints.
1452      */
1453     start_some_dumps(&runq);
1454 }
1455
1456
1457 static void
1458 handle_dumper_result(
1459     void *      cookie)
1460 {
1461     /*static int pending_aborts = 0;*/
1462     dumper_t *dumper = cookie;
1463     disk_t *dp, *sdp;
1464     cmd_t cmd;
1465     int result_argc;
1466     char *qname;
1467     char *result_argv[MAX_ARGS+1];
1468
1469     assert(dumper != NULL);
1470     dp = dumper->dp;
1471     assert(dp != NULL && sched(dp) != NULL);
1472
1473     do {
1474
1475         short_dump_state();
1476
1477         cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
1478
1479         if(cmd != BOGUS) {
1480             /* result_argv[2] always contains the serial number */
1481             sdp = serial2disk(result_argv[2]);
1482             if (sdp != dp) {
1483                 error("%s: Invalid serial number: %s", get_pname(), result_argv[2]);
1484                 /*NOTREACHED*/
1485             }
1486         }
1487
1488         qname = quote_string(dp->name);
1489         switch(cmd) {
1490
1491         case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
1492             if(result_argc != 6) {
1493                 error("error [dumper DONE result_argc != 6: %d]", result_argc);
1494                 /*NOTREACHED*/
1495             }
1496
1497             /*free_serial(result_argv[2]);*/
1498
1499             sched(dp)->origsize = OFF_T_ATOI(result_argv[3]);
1500             sched(dp)->dumptime = TIME_T_ATOI(result_argv[5]);
1501
1502             printf("driver: finished-cmd time %s %s dumped %s:%s\n",
1503                    walltime_str(curclock()), dumper->name,
1504                    dp->host->hostname, qname);
1505             fflush(stdout);
1506
1507             dumper->result = cmd;
1508
1509             break;
1510
1511         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
1512             /*
1513              * Requeue this disk, and fall through to the FAILED
1514              * case for cleanup.
1515              */
1516             if(sched(dp)->attempted) {
1517                 log_add(L_FAIL, "%s %s %s %d [too many dumper retry: %s]",
1518                     dp->host->hostname, dp->name, sched(dp)->datestamp,
1519                     sched(dp)->level, result_argv[3]);
1520                 printf("driver: dump failed %s %s %s, too many dumper retry: %s\n",
1521                         result_argv[2], dp->host->hostname, dp->name,
1522                         result_argv[3]);
1523             }
1524             /* FALLTHROUGH */
1525         case FAILED: /* FAILED <handle> <errstr> */
1526             /*free_serial(result_argv[2]);*/
1527             dumper->result = cmd;
1528             break;
1529
1530         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
1531             /*
1532              * We sent an ABORT from the NO-ROOM case because this dump
1533              * wasn't going to fit onto the holding disk.  We now need to
1534              * clean up the remains of this image, and try to finish
1535              * other dumps that are waiting on disk space.
1536              */
1537             assert(pending_aborts);
1538             /*free_serial(result_argv[2]);*/
1539             dumper->result = cmd;
1540             break;
1541
1542         case BOGUS:
1543             /* either EOF or garbage from dumper.  Turn it off */
1544             log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
1545                     dumper->name, (long)dumper->pid);
1546             if (dumper->ev_read) {
1547                 event_release(dumper->ev_read);
1548                 dumper->ev_read = NULL;
1549             }
1550             aclose(dumper->fd);
1551             dumper->busy = 0;
1552             dumper->down = 1;   /* mark it down so it isn't used again */
1553             if(dp) {
1554                 /* if it was dumping something, zap it and try again */
1555                 if(sched(dp)->attempted) {
1556                 log_add(L_FAIL, "%s %s %s %d [%s died]",
1557                         dp->host->hostname, qname, sched(dp)->datestamp,
1558                         sched(dp)->level, dumper->name);
1559                 }
1560                 else {
1561                 log_add(L_WARNING, "%s died while dumping %s:%s lev %d.",
1562                         dumper->name, dp->host->hostname, qname,
1563                         sched(dp)->level);
1564                 }
1565             }
1566             dumper->result = cmd;
1567             break;
1568
1569         default:
1570             assert(0);
1571         }
1572         amfree(qname);
1573
1574         /* send the dumper result to the chunker */
1575         if(dumper->chunker->down == 0 && dumper->chunker->fd != -1 &&
1576            dumper->chunker->result == LAST_TOK) {
1577             if(cmd == DONE) {
1578                 chunker_cmd(dumper->chunker, DONE, dp);
1579             }
1580             else {
1581                 chunker_cmd(dumper->chunker, FAILED, dp);
1582             }
1583         }
1584
1585         if(dumper->result != LAST_TOK && dumper->chunker->result != LAST_TOK)
1586             dumper_result(dp);
1587
1588     } while(areads_dataready(dumper->fd));
1589 }
1590
1591
1592 static void
1593 handle_chunker_result(
1594     void *      cookie)
1595 {
1596     /*static int pending_aborts = 0;*/
1597     chunker_t *chunker = cookie;
1598     assignedhd_t **h=NULL;
1599     dumper_t *dumper;
1600     disk_t *dp, *sdp;
1601     cmd_t cmd;
1602     int result_argc;
1603     char *result_argv[MAX_ARGS+1];
1604     int dummy;
1605     int activehd = -1;
1606     char *qname;
1607
1608     assert(chunker != NULL);
1609     dumper = chunker->dumper;
1610     assert(dumper != NULL);
1611     dp = dumper->dp;
1612     assert(dp != NULL);
1613     assert(sched(dp) != NULL);
1614     assert(sched(dp)->destname != NULL);
1615     assert(dp != NULL && sched(dp) != NULL && sched(dp)->destname);
1616
1617     if(dp && sched(dp) && sched(dp)->holdp) {
1618         h = sched(dp)->holdp;
1619         activehd = sched(dp)->activehd;
1620     }
1621
1622     do {
1623
1624         short_dump_state();
1625
1626         cmd = getresult(chunker->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
1627
1628         if(cmd != BOGUS) {
1629             /* result_argv[2] always contains the serial number */
1630             sdp = serial2disk(result_argv[2]);
1631             if (sdp != dp) {
1632                 error("%s: Invalid serial number: %s", get_pname(), result_argv[2]);
1633                 /*NOTREACHED*/
1634             }
1635         }
1636
1637         switch(cmd) {
1638
1639         case PARTIAL: /* PARTIAL <handle> <dumpsize> <errstr> */
1640         case DONE: /* DONE <handle> <dumpsize> <errstr> */
1641             if(result_argc != 4) {
1642                 error("error [chunker %s result_argc != 4: %d]", cmdstr[cmd],
1643                       result_argc);
1644                 /*NOTREACHED*/
1645             }
1646             /*free_serial(result_argv[2]);*/
1647
1648             sched(dp)->dumpsize = (off_t)atof(result_argv[3]);
1649
1650             qname = quote_string(dp->name);
1651             printf("driver: finished-cmd time %s %s chunked %s:%s\n",
1652                    walltime_str(curclock()), chunker->name,
1653                    dp->host->hostname, qname);
1654             fflush(stdout);
1655             amfree(qname);
1656
1657             event_release(chunker->ev_read);
1658
1659             chunker->result = cmd;
1660
1661             break;
1662
1663         case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
1664             event_release(chunker->ev_read);
1665
1666             chunker->result = cmd;
1667
1668             break;
1669         case FAILED: /* FAILED <handle> <errstr> */
1670             /*free_serial(result_argv[2]);*/
1671
1672             event_release(chunker->ev_read);
1673
1674             chunker->result = cmd;
1675
1676             break;
1677
1678         case NO_ROOM: /* NO-ROOM <handle> <missing_size> */
1679             if (!h || activehd < 0) { /* should never happen */
1680                 error("!h || activehd < 0");
1681                 /*NOTREACHED*/
1682             }
1683             h[activehd]->used -= OFF_T_ATOI(result_argv[3]);
1684             h[activehd]->reserved -= OFF_T_ATOI(result_argv[3]);
1685             holdalloc(h[activehd]->disk)->allocated_space -= OFF_T_ATOI(result_argv[3]);
1686             h[activehd]->disk->disksize -= OFF_T_ATOI(result_argv[3]);
1687             break;
1688
1689         case RQ_MORE_DISK: /* RQ-MORE-DISK <handle> */
1690             if (!h || activehd < 0) { /* should never happen */
1691                 error("!h || activehd < 0");
1692                 /*NOTREACHED*/
1693             }
1694             holdalloc(h[activehd]->disk)->allocated_dumpers--;
1695             h[activehd]->used = h[activehd]->reserved;
1696             if( h[++activehd] ) { /* There's still some allocated space left.
1697                                    * Tell the dumper about it. */
1698                 sched(dp)->activehd++;
1699                 chunker_cmd( chunker, CONTINUE, dp );
1700             } else { /* !h[++activehd] - must allocate more space */
1701                 sched(dp)->act_size = sched(dp)->est_size; /* not quite true */
1702                 sched(dp)->est_size = (sched(dp)->act_size/(off_t)20) * (off_t)21; /* +5% */
1703                 sched(dp)->est_size = am_round(sched(dp)->est_size, (off_t)DISK_BLOCK_KB);
1704                 if (sched(dp)->est_size < sched(dp)->act_size + 2*DISK_BLOCK_KB)
1705                     sched(dp)->est_size += 2 * DISK_BLOCK_KB;
1706                 h = find_diskspace( sched(dp)->est_size - sched(dp)->act_size,
1707                                     &dummy,
1708                                     h[activehd-1] );
1709                 if( !h ) {
1710                     /* No diskspace available. The reason for this will be
1711                      * determined in continue_port_dumps(). */
1712                     enqueue_disk( &roomq, dp );
1713                     continue_port_dumps();
1714                 } else {
1715                     /* OK, allocate space for disk and have chunker continue */
1716                     sched(dp)->activehd = assign_holdingdisk( h, dp );
1717                     chunker_cmd( chunker, CONTINUE, dp );
1718                     amfree(h);
1719                 }
1720             }
1721             break;
1722
1723         case ABORT_FINISHED: /* ABORT-FINISHED <handle> */
1724             /*
1725              * We sent an ABORT from the NO-ROOM case because this dump
1726              * wasn't going to fit onto the holding disk.  We now need to
1727              * clean up the remains of this image, and try to finish
1728              * other dumps that are waiting on disk space.
1729              */
1730             /*assert(pending_aborts);*/
1731
1732             /*free_serial(result_argv[2]);*/
1733
1734             event_release(chunker->ev_read);
1735
1736             chunker->result = cmd;
1737
1738             break;
1739
1740         case BOGUS:
1741             /* either EOF or garbage from chunker.  Turn it off */
1742             log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
1743                     chunker->name, (long)chunker->pid);
1744
1745             if(dp) {
1746                 /* if it was dumping something, zap it and try again */
1747                 if (!h || activehd < 0) { /* should never happen */
1748                     error("!h || activehd < 0");
1749                     /*NOTREACHED*/
1750                 }
1751                 qname = quote_string(dp->name);
1752                 if(sched(dp)->attempted) {
1753                     log_add(L_FAIL, "%s %s %s %d [%s died]",
1754                             dp->host->hostname, qname, sched(dp)->datestamp,
1755                             sched(dp)->level, chunker->name);
1756                 }
1757                 else {
1758                     log_add(L_WARNING, "%s died while dumping %s:%s lev %d.",
1759                             chunker->name, dp->host->hostname, qname,
1760                             sched(dp)->level);
1761                 }
1762                 amfree(qname);
1763                 dp = NULL;
1764             }
1765
1766             event_release(chunker->ev_read);
1767
1768             chunker->result = cmd;
1769
1770             break;
1771
1772         default:
1773             assert(0);
1774         }
1775
1776         if(chunker->result != LAST_TOK && chunker->dumper->result != LAST_TOK)
1777             dumper_result(dp);
1778
1779     } while(areads_dataready(chunker->fd));
1780 }
1781
1782
1783 static disklist_t
1784 read_flush(void)
1785 {
1786     sched_t *sp;
1787     disk_t *dp;
1788     int line;
1789     dumpfile_t file;
1790     char *hostname, *diskname, *datestamp;
1791     int level;
1792     char *destname;
1793     disk_t *dp1;
1794     char *inpline = NULL;
1795     char *command;
1796     char *s;
1797     int ch;
1798     disklist_t tq;
1799     char *qname = NULL;
1800     char *qdestname = NULL;
1801
1802     tq.head = tq.tail = NULL;
1803
1804     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
1805         line++;
1806         if (inpline[0] == '\0')
1807             continue;
1808
1809         s = inpline;
1810         ch = *s++;
1811
1812         skip_whitespace(s, ch);                 /* find the command */
1813         if(ch == '\0') {
1814             error("flush line %d: syntax error (no command)", line);
1815             /*NOTREACHED*/
1816         }
1817         command = s - 1;
1818         skip_non_whitespace(s, ch);
1819         s[-1] = '\0';
1820
1821         if(strcmp(command,"ENDFLUSH") == 0) {
1822             break;
1823         }
1824
1825         if(strcmp(command,"FLUSH") != 0) {
1826             error("flush line %d: syntax error (%s != FLUSH)", line, command);
1827             /*NOTREACHED*/
1828         }
1829
1830         skip_whitespace(s, ch);                 /* find the hostname */
1831         if(ch == '\0') {
1832             error("flush line %d: syntax error (no hostname)", line);
1833             /*NOTREACHED*/
1834         }
1835         hostname = s - 1;
1836         skip_non_whitespace(s, ch);
1837         s[-1] = '\0';
1838
1839         skip_whitespace(s, ch);                 /* find the diskname */
1840         if(ch == '\0') {
1841             error("flush line %d: syntax error (no diskname)", line);
1842             /*NOTREACHED*/
1843         }
1844         qname = s - 1;
1845         skip_quoted_string(s, ch);
1846         s[-1] = '\0';                           /* terminate the disk name */
1847         diskname = unquote_string(qname);
1848
1849         skip_whitespace(s, ch);                 /* find the datestamp */
1850         if(ch == '\0') {
1851             error("flush line %d: syntax error (no datestamp)", line);
1852             /*NOTREACHED*/
1853         }
1854         datestamp = s - 1;
1855         skip_non_whitespace(s, ch);
1856         s[-1] = '\0';
1857
1858         skip_whitespace(s, ch);                 /* find the level number */
1859         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
1860             error("flush line %d: syntax error (bad level)", line);
1861             /*NOTREACHED*/
1862         }
1863         skip_integer(s, ch);
1864
1865         skip_whitespace(s, ch);                 /* find the filename */
1866         if(ch == '\0') {
1867             error("flush line %d: syntax error (no filename)", line);
1868             /*NOTREACHED*/
1869         }
1870         qdestname = s - 1;
1871         skip_quoted_string(s, ch);
1872         s[-1] = '\0';
1873         destname = unquote_string(qdestname);
1874
1875         holding_file_get_dumpfile(destname, &file);
1876         if( file.type != F_DUMPFILE) {
1877             if( file.type != F_CONT_DUMPFILE )
1878                 log_add(L_INFO, "%s: ignoring cruft file.", destname);
1879             amfree(diskname);
1880             continue;
1881         }
1882
1883         if(strcmp(hostname, file.name) != 0 ||
1884            strcmp(diskname, file.disk) != 0 ||
1885            strcmp(datestamp, file.datestamp) != 0) {
1886             log_add(L_INFO, "disk %s:%s not consistent with file %s",
1887                     hostname, diskname, destname);
1888             amfree(diskname);
1889             continue;
1890         }
1891         amfree(diskname);
1892
1893         dp = lookup_disk(file.name, file.disk);
1894
1895         if (dp == NULL) {
1896             log_add(L_INFO, "%s: disk %s:%s not in database, skipping it.",
1897                     destname, file.name, file.disk);
1898             continue;
1899         }
1900
1901         if(file.dumplevel < 0 || file.dumplevel > 9) {
1902             log_add(L_INFO, "%s: ignoring file with bogus dump level %d.",
1903                     destname, file.dumplevel);
1904             continue;
1905         }
1906
1907         if (holding_file_size(destname,1) <= 0) {
1908             log_add(L_INFO, "%s: removing file with no data.", destname);
1909             holding_file_unlink(destname);
1910             continue;
1911         }
1912
1913         dp1 = (disk_t *)alloc(SIZEOF(disk_t));
1914         *dp1 = *dp;
1915         dp1->next = dp1->prev = NULL;
1916
1917         /* add it to the flushhost list */
1918         if(!flushhost) {
1919             flushhost = alloc(SIZEOF(am_host_t));
1920             flushhost->next = NULL;
1921             flushhost->hostname = stralloc("FLUSHHOST");
1922             flushhost->up = NULL;
1923             flushhost->features = NULL;
1924         }
1925         dp1->hostnext = flushhost->disks;
1926         flushhost->disks = dp1;
1927
1928         sp = (sched_t *) alloc(SIZEOF(sched_t));
1929         sp->destname = stralloc(destname);
1930         sp->level = file.dumplevel;
1931         sp->dumpdate = NULL;
1932         sp->degr_dumpdate = NULL;
1933         sp->datestamp = stralloc(file.datestamp);
1934         sp->est_nsize = (off_t)0;
1935         sp->est_csize = (off_t)0;
1936         sp->est_time = 0;
1937         sp->est_kps = 10;
1938         sp->priority = 0;
1939         sp->degr_level = -1;
1940         sp->attempted = 0;
1941         sp->act_size = holding_file_size(destname, 0);
1942         sp->holdp = build_diskspace(destname);
1943         if(sp->holdp == NULL) continue;
1944         sp->dumper = NULL;
1945         sp->timestamp = (time_t)0;
1946
1947         dp1->up = (char *)sp;
1948
1949         enqueue_disk(&tq, dp1);
1950     }
1951     amfree(inpline);
1952
1953     /*@i@*/ return tq;
1954 }
1955
1956 static void
1957 read_schedule(
1958     void *      cookie)
1959 {
1960     sched_t *sp;
1961     disk_t *dp;
1962     int level, line, priority;
1963     char *dumpdate, *degr_dumpdate;
1964     int degr_level;
1965     time_t time, degr_time;
1966     time_t *time_p = &time;
1967     time_t *degr_time_p = &degr_time;
1968     off_t nsize, csize, degr_nsize, degr_csize;
1969     unsigned long kps, degr_kps;
1970     char *hostname, *features, *diskname, *datestamp, *inpline = NULL;
1971     char *command;
1972     char *s;
1973     int ch;
1974     off_t flush_size = (off_t)0;
1975     char *qname = NULL;
1976     OFF_T_FMT_TYPE nsize_;
1977     OFF_T_FMT_TYPE csize_;
1978     OFF_T_FMT_TYPE degr_nsize_;
1979     OFF_T_FMT_TYPE degr_csize_;
1980
1981     (void)cookie;       /* Quiet unused parameter warning */
1982
1983     event_release(schedule_ev_read);
1984
1985     /* read schedule from stdin */
1986
1987     for(line = 0; (inpline = agets(stdin)) != NULL; free(inpline)) {
1988         if (inpline[0] == '\0')
1989             continue;
1990         line++;
1991
1992         s = inpline;
1993         ch = *s++;
1994
1995         skip_whitespace(s, ch);                 /* find the command */
1996         if(ch == '\0') {
1997             error("schedule line %d: syntax error (no command)", line);
1998             /*NOTREACHED*/
1999         }
2000         command = s - 1;
2001         skip_non_whitespace(s, ch);
2002         s[-1] = '\0';
2003
2004         if(strcmp(command,"DUMP") != 0) {
2005             error("schedule line %d: syntax error (%s != DUMP)", line, command);
2006             /*NOTREACHED*/
2007         }
2008
2009         skip_whitespace(s, ch);                 /* find the host name */
2010         if(ch == '\0') {
2011             error("schedule line %d: syntax error (no host name)", line);
2012             /*NOTREACHED*/
2013         }
2014         hostname = s - 1;
2015         skip_non_whitespace(s, ch);
2016         s[-1] = '\0';
2017
2018         skip_whitespace(s, ch);                 /* find the feature list */
2019         if(ch == '\0') {
2020             error("schedule line %d: syntax error (no feature list)", line);
2021             /*NOTREACHED*/
2022         }
2023         features = s - 1;
2024         skip_non_whitespace(s, ch);
2025         s[-1] = '\0';
2026
2027         skip_whitespace(s, ch);                 /* find the disk name */
2028         if(ch == '\0') {
2029             error("schedule line %d: syntax error (no disk name)", line);
2030             /*NOTREACHED*/
2031         }
2032         qname = s - 1;
2033         skip_quoted_string(s, ch);
2034         s[-1] = '\0';                           /* terminate the disk name */
2035         diskname = unquote_string(qname);
2036
2037         skip_whitespace(s, ch);                 /* find the datestamp */
2038         if(ch == '\0') {
2039             error("schedule line %d: syntax error (no datestamp)", line);
2040             /*NOTREACHED*/
2041         }
2042         datestamp = s - 1;
2043         skip_non_whitespace(s, ch);
2044         s[-1] = '\0';
2045
2046         skip_whitespace(s, ch);                 /* find the priority number */
2047         if(ch == '\0' || sscanf(s - 1, "%d", &priority) != 1) {
2048             error("schedule line %d: syntax error (bad priority)", line);
2049             /*NOTREACHED*/
2050         }
2051         skip_integer(s, ch);
2052
2053         skip_whitespace(s, ch);                 /* find the level number */
2054         if(ch == '\0' || sscanf(s - 1, "%d", &level) != 1) {
2055             error("schedule line %d: syntax error (bad level)", line);
2056             /*NOTREACHED*/
2057         }
2058         skip_integer(s, ch);
2059
2060         skip_whitespace(s, ch);                 /* find the dump date */
2061         if(ch == '\0') {
2062             error("schedule line %d: syntax error (bad dump date)", line);
2063             /*NOTREACHED*/
2064         }
2065         dumpdate = s - 1;
2066         skip_non_whitespace(s, ch);
2067         s[-1] = '\0';
2068
2069         skip_whitespace(s, ch);                 /* find the native size */
2070         nsize_ = (OFF_T_FMT_TYPE)0;
2071         if(ch == '\0' || sscanf(s - 1, OFF_T_FMT, &nsize_) != 1) {
2072             error("schedule line %d: syntax error (bad nsize)", line);
2073             /*NOTREACHED*/
2074         }
2075         nsize = (off_t)nsize_;
2076         skip_integer(s, ch);
2077
2078         skip_whitespace(s, ch);                 /* find the compressed size */
2079         csize_ = (OFF_T_FMT_TYPE)0;
2080         if(ch == '\0' || sscanf(s - 1, OFF_T_FMT, &csize_) != 1) {
2081             error("schedule line %d: syntax error (bad csize)", line);
2082             /*NOTREACHED*/
2083         }
2084         csize = (off_t)csize_;
2085         skip_integer(s, ch);
2086
2087         skip_whitespace(s, ch);                 /* find the time number */
2088         if(ch == '\0' || sscanf(s - 1, TIME_T_FMT,
2089                                 (TIME_T_FMT_TYPE *)time_p) != 1) {
2090             error("schedule line %d: syntax error (bad estimated time)", line);
2091             /*NOTREACHED*/
2092         }
2093         skip_integer(s, ch);
2094
2095         skip_whitespace(s, ch);                 /* find the kps number */
2096         if(ch == '\0' || sscanf(s - 1, "%lu", &kps) != 1) {
2097             error("schedule line %d: syntax error (bad kps)", line);
2098             continue;
2099         }
2100         skip_integer(s, ch);
2101
2102         degr_dumpdate = NULL;                   /* flag if degr fields found */
2103         skip_whitespace(s, ch);                 /* find the degr level number */
2104         if(ch != '\0') {
2105             if(sscanf(s - 1, "%d", &degr_level) != 1) {
2106                 error("schedule line %d: syntax error (bad degr level)", line);
2107                 /*NOTREACHED*/
2108             }
2109             skip_integer(s, ch);
2110
2111             skip_whitespace(s, ch);             /* find the degr dump date */
2112             if(ch == '\0') {
2113                 error("schedule line %d: syntax error (bad degr dump date)", line);
2114                 /*NOTREACHED*/
2115             }
2116             degr_dumpdate = s - 1;
2117             skip_non_whitespace(s, ch);
2118             s[-1] = '\0';
2119
2120             skip_whitespace(s, ch);             /* find the degr native size */
2121             degr_nsize_ = (OFF_T_FMT_TYPE)0;
2122             if(ch == '\0'  || sscanf(s - 1, OFF_T_FMT, &degr_nsize_) != 1) {
2123                 error("schedule line %d: syntax error (bad degr nsize)", line);
2124                 /*NOTREACHED*/
2125             }
2126             degr_nsize = (off_t)degr_nsize_;
2127             skip_integer(s, ch);
2128
2129             skip_whitespace(s, ch);             /* find the degr compressed size */
2130             degr_csize_ = (OFF_T_FMT_TYPE)0;
2131             if(ch == '\0'  || sscanf(s - 1, OFF_T_FMT, &degr_csize_) != 1) {
2132                 error("schedule line %d: syntax error (bad degr csize)", line);
2133                 /*NOTREACHED*/
2134             }
2135             degr_csize = (off_t)degr_csize_;
2136             skip_integer(s, ch);
2137
2138             skip_whitespace(s, ch);             /* find the degr time number */
2139             if(ch == '\0' || sscanf(s - 1, TIME_T_FMT,
2140                                 (TIME_T_FMT_TYPE *)degr_time_p) != 1) {
2141                 error("schedule line %d: syntax error (bad degr estimated time)", line);
2142                 /*NOTREACHED*/
2143             }
2144             skip_integer(s, ch);
2145
2146             skip_whitespace(s, ch);             /* find the degr kps number */
2147             if(ch == '\0' || sscanf(s - 1, "%lu", &degr_kps) != 1) {
2148                 error("schedule line %d: syntax error (bad degr kps)", line);
2149                 /*NOTREACHED*/
2150             }
2151             skip_integer(s, ch);
2152         } else {
2153             degr_level = -1;
2154             degr_nsize = (off_t)0;
2155             degr_csize = (off_t)0;
2156             degr_time = (time_t)0;
2157             degr_kps = 0;
2158         }
2159
2160         dp = lookup_disk(hostname, diskname);
2161         if(dp == NULL) {
2162             log_add(L_WARNING,
2163                     "schedule line %d: %s:'%s' not in disklist, ignored",
2164                     line, hostname, qname);
2165             amfree(diskname);
2166             continue;
2167         }
2168
2169         sp = (sched_t *) alloc(SIZEOF(sched_t));
2170         /*@ignore@*/
2171         sp->level = level;
2172         sp->dumpdate = stralloc(dumpdate);
2173         sp->est_nsize = DISK_BLOCK_KB + nsize; /* include header */
2174         sp->est_csize = DISK_BLOCK_KB + csize; /* include header */
2175         /* round estimate to next multiple of DISK_BLOCK_KB */
2176         sp->est_csize = am_round(sp->est_csize, DISK_BLOCK_KB);
2177         sp->est_size = sp->est_csize;
2178         sp->est_time = time;
2179         sp->est_kps = kps;
2180         sp->priority = priority;
2181         sp->datestamp = stralloc(datestamp);
2182
2183         if(degr_dumpdate) {
2184             sp->degr_level = degr_level;
2185             sp->degr_dumpdate = stralloc(degr_dumpdate);
2186             sp->degr_nsize = DISK_BLOCK_KB + degr_nsize;
2187             sp->degr_csize = DISK_BLOCK_KB + degr_csize;
2188             /* round estimate to next multiple of DISK_BLOCK_KB */
2189             sp->degr_csize = am_round(sp->degr_csize, DISK_BLOCK_KB);
2190             sp->degr_time = degr_time;
2191             sp->degr_kps = degr_kps;
2192         } else {
2193             sp->degr_level = -1;
2194             sp->degr_dumpdate = NULL;
2195         }
2196         /*@end@*/
2197
2198         sp->attempted = 0;
2199         sp->act_size = (off_t)0;
2200         sp->holdp = NULL;
2201         sp->activehd = -1;
2202         sp->dumper = NULL;
2203         sp->timestamp = (time_t)0;
2204         sp->destname = NULL;
2205         sp->no_space = 0;
2206
2207         dp->up = (char *) sp;
2208         if(dp->host->features == NULL) {
2209             dp->host->features = am_string_to_feature(features);
2210         }
2211         remove_disk(&waitq, dp);
2212         enqueue_disk(&runq, dp);
2213         flush_size += sp->act_size;
2214         amfree(diskname);
2215     }
2216     printf("driver: flush size " OFF_T_FMT "\n", (OFF_T_FMT_TYPE)flush_size);
2217     amfree(inpline);
2218     if(line == 0)
2219         log_add(L_WARNING, "WARNING: got empty schedule from planner");
2220     if(need_degraded==1) start_degraded_mode(&runq);
2221     start_some_dumps(&runq);
2222 }
2223
2224 static unsigned long
2225 free_kps(
2226     interface_t *ip)
2227 {
2228     unsigned long res;
2229
2230     if (ip == (interface_t *)0) {
2231         interface_t *p;
2232         unsigned long maxusage=0;
2233         unsigned long curusage=0;
2234         for(p = lookup_interface(NULL); p != NULL; p = p->next) {
2235             maxusage += interface_get_maxusage(p);
2236             curusage += p->curusage;
2237         }
2238         if (maxusage >= curusage)
2239             res = maxusage - curusage;
2240         else
2241             res = 0;
2242 #ifndef __lint
2243     } else {
2244         if ((unsigned long)interface_get_maxusage(ip) >= ip->curusage)
2245             res = interface_get_maxusage(ip) - ip->curusage;
2246         else
2247             res = 0;
2248 #endif
2249     }
2250
2251     return res;
2252 }
2253
2254 static void
2255 interface_state(
2256     char *time_str)
2257 {
2258     interface_t *ip;
2259
2260     printf("driver: interface-state time %s", time_str);
2261
2262     for(ip = lookup_interface(NULL); ip != NULL; ip = ip->next) {
2263         printf(" if %s: free %lu", ip->name, free_kps(ip));
2264     }
2265     printf("\n");
2266 }
2267
2268 static void
2269 allocate_bandwidth(
2270     interface_t *       ip,
2271     unsigned long       kps)
2272 {
2273     ip->curusage += kps;
2274 }
2275
2276 static void
2277 deallocate_bandwidth(
2278     interface_t *       ip,
2279     unsigned long       kps)
2280 {
2281     assert(kps <= ip->curusage);
2282     ip->curusage -= kps;
2283 }
2284
2285 /* ------------ */
2286 static off_t
2287 free_space(void)
2288 {
2289     holdingdisk_t *hdp;
2290     off_t total_free;
2291     off_t diff;
2292
2293     total_free = (off_t)0;
2294     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2295         diff = hdp->disksize - holdalloc(hdp)->allocated_space;
2296         if(diff > (off_t)0)
2297             total_free += diff;
2298     }
2299     return total_free;
2300 }
2301
2302 /*
2303  * We return an array of pointers to assignedhd_t. The array contains at
2304  * most one entry per holding disk. The list of pointers is terminated by
2305  * a NULL pointer. Each entry contains a pointer to a holdingdisk and
2306  * how much diskspace to use on that disk. Later on, assign_holdingdisk
2307  * will allocate the given amount of space.
2308  * If there is not enough room on the holdingdisks, NULL is returned.
2309  */
2310
2311 static assignedhd_t **
2312 find_diskspace(
2313     off_t               size,
2314     int *               cur_idle,
2315     assignedhd_t *      pref)
2316 {
2317     assignedhd_t **result = NULL;
2318     holdingdisk_t *minp, *hdp;
2319     int i=0, num_holdingdisks=0; /* are we allowed to use the global thing? */
2320     int j, minj;
2321     char *used;
2322     off_t halloc, dalloc, hfree, dfree;
2323
2324     (void)cur_idle;     /* Quiet unused parameter warning */
2325
2326     if (size < 2*DISK_BLOCK_KB)
2327         size = 2*DISK_BLOCK_KB;
2328     size = am_round(size, (off_t)DISK_BLOCK_KB);
2329
2330     hold_debug(1, ("%s: want " OFF_T_FMT " K\n",
2331                    debug_prefix_time(": find_diskspace"),
2332                    (OFF_T_FMT_TYPE)size));
2333
2334     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2335         num_holdingdisks++;
2336     }
2337
2338     used = alloc(SIZEOF(*used) * num_holdingdisks);/*disks used during this run*/
2339     memset( used, 0, (size_t)num_holdingdisks );
2340     result = alloc(SIZEOF(assignedhd_t *) * (num_holdingdisks + 1));
2341     result[0] = NULL;
2342
2343     while( i < num_holdingdisks && size > (off_t)0 ) {
2344         /* find the holdingdisk with the fewest active dumpers and among
2345          * those the one with the biggest free space
2346          */
2347         minp = NULL; minj = -1;
2348         for(j = 0, hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next, j++ ) {
2349             if( pref && pref->disk == hdp && !used[j] &&
2350                 holdalloc(hdp)->allocated_space <= hdp->disksize - (off_t)DISK_BLOCK_KB) {
2351                 minp = hdp;
2352                 minj = j;
2353                 break;
2354             }
2355             else if( holdalloc(hdp)->allocated_space <= hdp->disksize - (off_t)(2*DISK_BLOCK_KB) &&
2356                 !used[j] &&
2357                 (!minp ||
2358                  holdalloc(hdp)->allocated_dumpers < holdalloc(minp)->allocated_dumpers ||
2359                  (holdalloc(hdp)->allocated_dumpers == holdalloc(minp)->allocated_dumpers &&
2360                   hdp->disksize-holdalloc(hdp)->allocated_space > minp->disksize-holdalloc(minp)->allocated_space)) ) {
2361                 minp = hdp;
2362                 minj = j;
2363             }
2364         }
2365
2366         pref = NULL;
2367         if( !minp ) { break; } /* all holding disks are full */
2368         used[minj] = 1;
2369
2370         /* hfree = free space on the disk */
2371         hfree = minp->disksize - holdalloc(minp)->allocated_space;
2372
2373         /* dfree = free space for data, remove 1 header for each chunksize */
2374         dfree = hfree - (((hfree-(off_t)1)/holdingdisk_get_chunksize(minp))+(off_t)1) * (off_t)DISK_BLOCK_KB;
2375
2376         /* dalloc = space I can allocate for data */
2377         dalloc = ( dfree < size ) ? dfree : size;
2378
2379         /* halloc = space to allocate, including 1 header for each chunksize */
2380         halloc = dalloc + (((dalloc-(off_t)1)/holdingdisk_get_chunksize(minp))+(off_t)1) * (off_t)DISK_BLOCK_KB;
2381
2382         hold_debug(1, ("%s: find diskspace: size " OFF_T_FMT " hf " OFF_T_FMT
2383                        " df " OFF_T_FMT " da " OFF_T_FMT " ha " OFF_T_FMT"\n",
2384                        debug_prefix_time(": find_diskspace"),
2385                        (OFF_T_FMT_TYPE)size,
2386                        (OFF_T_FMT_TYPE)hfree,
2387                        (OFF_T_FMT_TYPE)dfree,
2388                        (OFF_T_FMT_TYPE)dalloc,
2389                        (OFF_T_FMT_TYPE)halloc));
2390         size -= dalloc;
2391         result[i] = alloc(SIZEOF(assignedhd_t));
2392         result[i]->disk = minp;
2393         result[i]->reserved = halloc;
2394         result[i]->used = (off_t)0;
2395         result[i]->destname = NULL;
2396         result[i+1] = NULL;
2397         i++;
2398     } /* while i < num_holdingdisks && size > 0 */
2399     amfree(used);
2400
2401     if(size != (off_t)0) { /* not enough space available */
2402         printf("find diskspace: not enough diskspace. Left with "
2403                OFF_T_FMT " K\n", (OFF_T_FMT_TYPE)size);
2404         fflush(stdout);
2405         free_assignedhd(result);
2406         result = NULL;
2407     }
2408
2409     if (debug_holding > 1) {
2410         for( i = 0; result && result[i]; i++ ) {
2411             hold_debug(1, ("%s: find diskspace: selected %s free " OFF_T_FMT
2412                            " reserved " OFF_T_FMT " dumpers %d\n",
2413                            debug_prefix_time(": find_diskspace"),
2414                            holdingdisk_get_diskdir(result[i]->disk),
2415                            (OFF_T_FMT_TYPE)(result[i]->disk->disksize -
2416                              holdalloc(result[i]->disk)->allocated_space),
2417                            (OFF_T_FMT_TYPE)result[i]->reserved,
2418                            holdalloc(result[i]->disk)->allocated_dumpers));
2419         }
2420     }
2421
2422     return result;
2423 }
2424
2425 static int
2426 assign_holdingdisk(
2427     assignedhd_t **     holdp,
2428     disk_t *            diskp)
2429 {
2430     int i, j, c, l=0;
2431     off_t size;
2432     char *sfn = sanitise_filename(diskp->name);
2433     char lvl[64];
2434     assignedhd_t **new_holdp;
2435     char *qname;
2436
2437     snprintf( lvl, SIZEOF(lvl), "%d", sched(diskp)->level );
2438
2439     size = am_round(sched(diskp)->est_size - sched(diskp)->act_size,
2440                     (off_t)DISK_BLOCK_KB);
2441
2442     for( c = 0; holdp[c]; c++ )
2443         (void)c; /* count number of disks */
2444
2445     /* allocate memory for sched(diskp)->holdp */
2446     for(j = 0; sched(diskp)->holdp && sched(diskp)->holdp[j]; j++)
2447         (void)j;        /* Quiet lint */
2448     new_holdp = (assignedhd_t **)alloc(SIZEOF(assignedhd_t*)*(j+c+1));
2449     if (sched(diskp)->holdp) {
2450         memcpy(new_holdp, sched(diskp)->holdp, j * SIZEOF(*new_holdp));
2451         amfree(sched(diskp)->holdp);
2452     }
2453     sched(diskp)->holdp = new_holdp;
2454     new_holdp = NULL;
2455
2456     i = 0;
2457     if( j > 0 ) { /* This is a request for additional diskspace. See if we can
2458                    * merge assignedhd_t's */
2459         l=j;
2460         if( sched(diskp)->holdp[j-1]->disk == holdp[0]->disk ) { /* Yes! */
2461             sched(diskp)->holdp[j-1]->reserved += holdp[0]->reserved;
2462             holdalloc(holdp[0]->disk)->allocated_space += holdp[0]->reserved;
2463             size = (holdp[0]->reserved>size) ? (off_t)0 : size-holdp[0]->reserved;
2464             qname = quote_string(diskp->name);
2465             hold_debug(1, ("%s: merging holding disk %s to disk %s:%s, add "
2466                            OFF_T_FMT " for reserved " OFF_T_FMT ", left "
2467                            OFF_T_FMT "\n",
2468                            debug_prefix_time(": assign_holdingdisk"),
2469                            holdingdisk_get_diskdir(
2470                                                sched(diskp)->holdp[j-1]->disk),
2471                            diskp->host->hostname, qname,
2472                            (OFF_T_FMT_TYPE)holdp[0]->reserved,
2473                            (OFF_T_FMT_TYPE)sched(diskp)->holdp[j-1]->reserved,
2474                            (OFF_T_FMT_TYPE)size));
2475             i++;
2476             amfree(qname);
2477             amfree(holdp[0]);
2478             l=j-1;
2479         }
2480     }
2481
2482     /* copy assignedhd_s to sched(diskp), adjust allocated_space */
2483     for( ; holdp[i]; i++ ) {
2484         holdp[i]->destname = newvstralloc( holdp[i]->destname,
2485                                            holdingdisk_get_diskdir(holdp[i]->disk), "/",
2486                                            hd_driver_timestamp, "/",
2487                                            diskp->host->hostname, ".",
2488                                            sfn, ".",
2489                                            lvl, NULL );
2490         sched(diskp)->holdp[j++] = holdp[i];
2491         holdalloc(holdp[i]->disk)->allocated_space += holdp[i]->reserved;
2492         size = (holdp[i]->reserved > size) ? (off_t)0 :
2493                   (size - holdp[i]->reserved);
2494         qname = quote_string(diskp->name);
2495         hold_debug(1,
2496                    ("%s: %d assigning holding disk %s to disk %s:%s, reserved "
2497                     OFF_T_FMT ", left " OFF_T_FMT "\n",
2498                     debug_prefix_time(": assign_holdingdisk"),
2499                     i, holdingdisk_get_diskdir(holdp[i]->disk),
2500                     diskp->host->hostname, qname,
2501                     (OFF_T_FMT_TYPE)holdp[i]->reserved,
2502                     (OFF_T_FMT_TYPE)size));
2503         amfree(qname);
2504         holdp[i] = NULL; /* so it doesn't get free()d... */
2505     }
2506     sched(diskp)->holdp[j] = NULL;
2507     amfree(sfn);
2508
2509     return l;
2510 }
2511
2512 static void
2513 adjust_diskspace(
2514     disk_t *    diskp,
2515     cmd_t       cmd)
2516 {
2517     assignedhd_t **holdp;
2518     off_t total = (off_t)0;
2519     off_t diff;
2520     int i;
2521     char *qname, *hqname, *qdest;
2522
2523     (void)cmd;  /* Quiet unused parameter warning */
2524
2525     qname = quote_string(diskp->name);
2526     qdest = quote_string(sched(diskp)->destname);
2527     hold_debug(1, ("%s: %s:%s %s\n",
2528                    debug_prefix_time(": adjust_diskspace"),
2529                    diskp->host->hostname, qname, qdest));
2530
2531     holdp = sched(diskp)->holdp;
2532
2533     assert(holdp != NULL);
2534
2535     for( i = 0; holdp[i]; i++ ) { /* for each allocated disk */
2536         diff = holdp[i]->used - holdp[i]->reserved;
2537         total += holdp[i]->used;
2538         holdalloc(holdp[i]->disk)->allocated_space += diff;
2539         hqname = quote_string(holdp[i]->disk->name);
2540         hold_debug(1, ("%s: hdisk %s done, reserved " OFF_T_FMT " used "
2541                        OFF_T_FMT " diff " OFF_T_FMT " alloc " OFF_T_FMT
2542                        " dumpers %d\n",
2543                        debug_prefix_time(": adjust_diskspace"),
2544                        holdp[i]->disk->name,
2545                        (OFF_T_FMT_TYPE)holdp[i]->reserved,
2546                        (OFF_T_FMT_TYPE)holdp[i]->used,
2547                        (OFF_T_FMT_TYPE)diff,
2548                        (OFF_T_FMT_TYPE)holdalloc(holdp[i]->disk)
2549                                                              ->allocated_space,
2550                        holdalloc(holdp[i]->disk)->allocated_dumpers ));
2551         holdp[i]->reserved += diff;
2552         amfree(hqname);
2553     }
2554
2555     sched(diskp)->act_size = total;
2556
2557     hold_debug(1, ("%s: after: disk %s:%s used " OFF_T_FMT "\n",
2558                    debug_prefix_time(": adjust_diskspace"),
2559                    diskp->host->hostname, qname,
2560                    (OFF_T_FMT_TYPE)sched(diskp)->act_size));
2561     amfree(qdest);
2562     amfree(qname);
2563 }
2564
2565 static void
2566 delete_diskspace(
2567     disk_t *diskp)
2568 {
2569     assignedhd_t **holdp;
2570     int i;
2571
2572     holdp = sched(diskp)->holdp;
2573
2574     assert(holdp != NULL);
2575
2576     for( i = 0; holdp[i]; i++ ) { /* for each disk */
2577         /* find all files of this dump on that disk, and subtract their
2578          * reserved sizes from the disk's allocated space
2579          */
2580         holdalloc(holdp[i]->disk)->allocated_space -= holdp[i]->used;
2581     }
2582
2583     holding_file_unlink(holdp[0]->destname);    /* no need for the entire list,
2584                                                  * because holding_file_unlink
2585                                                  * will walk through all files
2586                                                  * using cont_filename */
2587     free_assignedhd(sched(diskp)->holdp);
2588     sched(diskp)->holdp = NULL;
2589     sched(diskp)->act_size = (off_t)0;
2590 }
2591
2592 static assignedhd_t **
2593 build_diskspace(
2594     char *      destname)
2595 {
2596     int i, j;
2597     int fd;
2598     ssize_t buflen;
2599     char buffer[DISK_BLOCK_BYTES];
2600     dumpfile_t file;
2601     assignedhd_t **result;
2602     holdingdisk_t *hdp;
2603     off_t *used;
2604     int num_holdingdisks=0;
2605     char dirname[1000], *ch;
2606     struct stat finfo;
2607     char *filename = destname;
2608
2609     memset(buffer, 0, sizeof(buffer));
2610     for(hdp = getconf_holdingdisks(); hdp != NULL; hdp = hdp->next) {
2611         num_holdingdisks++;
2612     }
2613     used = alloc(SIZEOF(off_t) * num_holdingdisks);
2614     for(i=0;i<num_holdingdisks;i++)
2615         used[i] = (off_t)0;
2616     result = alloc(SIZEOF(assignedhd_t *) * (num_holdingdisks + 1));
2617     result[0] = NULL;
2618     while(filename != NULL && filename[0] != '\0') {
2619         strncpy(dirname, filename, 999);
2620         dirname[999]='\0';
2621         ch = strrchr(dirname,'/');
2622         *ch = '\0';
2623         ch = strrchr(dirname,'/');
2624         *ch = '\0';
2625
2626         for(j = 0, hdp = getconf_holdingdisks(); hdp != NULL;
2627                                                  hdp = hdp->next, j++ ) {
2628             if(strcmp(dirname, holdingdisk_get_diskdir(hdp))==0) {
2629                 break;
2630             }
2631         }
2632
2633         if(stat(filename, &finfo) == -1) {
2634             fprintf(stderr, "stat %s: %s\n", filename, strerror(errno));
2635             finfo.st_size = (off_t)0;
2636         }
2637         used[j] += ((off_t)finfo.st_size+(off_t)1023)/(off_t)1024;
2638         if((fd = open(filename,O_RDONLY)) == -1) {
2639             fprintf(stderr,"build_diskspace: open of %s failed: %s\n",
2640                     filename, strerror(errno));
2641             return NULL;
2642         }
2643         if ((buflen = fullread(fd, buffer, SIZEOF(buffer))) > 0) {;
2644                 parse_file_header(buffer, &file, (size_t)buflen);
2645         }
2646         close(fd);
2647         filename = file.cont_filename;
2648     }
2649
2650     for(j = 0, i=0, hdp = getconf_holdingdisks(); hdp != NULL;
2651                                                   hdp = hdp->next, j++ ) {
2652         if(used[j] != (off_t)0) {
2653             result[i] = alloc(SIZEOF(assignedhd_t));
2654             result[i]->disk = hdp;
2655             result[i]->reserved = used[j];
2656             result[i]->used = used[j];
2657             result[i]->destname = stralloc(destname);
2658             result[i+1] = NULL;
2659             i++;
2660         }
2661     }
2662
2663     amfree(used);
2664     return result;
2665 }
2666
2667 static void
2668 holdingdisk_state(
2669     char *      time_str)
2670 {
2671     holdingdisk_t *hdp;
2672     int dsk;
2673     off_t diff;
2674
2675     printf("driver: hdisk-state time %s", time_str);
2676
2677     for(hdp = getconf_holdingdisks(), dsk = 0; hdp != NULL; hdp = hdp->next, dsk++) {
2678         diff = hdp->disksize - holdalloc(hdp)->allocated_space;
2679         printf(" hdisk %d: free " OFF_T_FMT " dumpers %d", dsk,
2680                (OFF_T_FMT_TYPE)diff, holdalloc(hdp)->allocated_dumpers);
2681     }
2682     printf("\n");
2683 }
2684
2685 static void
2686 update_failed_dump_to_tape(
2687     disk_t *    dp)
2688 {
2689 /* JLM
2690  * should simply set no_bump
2691  */
2692
2693     time_t save_timestamp = sched(dp)->timestamp;
2694     /* setting timestamp to 0 removes the current level from the
2695      * database, so that we ensure that it will not be bumped to the
2696      * next level on the next run.  If we didn't do this, dumpdates or
2697      * gnutar-lists might have been updated already, and a bumped
2698      * incremental might be created.  */
2699     sched(dp)->timestamp = 0;
2700     update_info_dumper(dp, (off_t)-1, (off_t)-1, (time_t)-1);
2701     sched(dp)->timestamp = save_timestamp;
2702 }
2703
2704 /* ------------------- */
2705 static int
2706 dump_to_tape(
2707     disk_t *    dp)
2708 {
2709     dumper_t *dumper;
2710     int failed = 0;
2711     off_t filenum;
2712     off_t origsize = (off_t)0;
2713     off_t dumpsize = (off_t)0;
2714     time_t dumptime = (time_t)0;
2715     double tapetime = 0.0;
2716     cmd_t cmd;
2717     int result_argc, rc;
2718     char *result_argv[MAX_ARGS+1];
2719     int dumper_tryagain = 0;
2720     char *qname;
2721
2722     qname = quote_string(dp->name);
2723     printf("driver: dumping %s:%s directly to tape\n",
2724            dp->host->hostname, qname);
2725     fflush(stdout);
2726
2727     /* pick a dumper and fail if there are no idle dumpers */
2728
2729     dumper = idle_dumper();
2730     if (!dumper) {
2731         printf("driver: no idle dumpers for %s:%s.\n",
2732                 dp->host->hostname, qname);
2733         fflush(stdout);
2734         log_add(L_WARNING, "no idle dumpers for %s:%s.\n",
2735                 dp->host->hostname, qname);
2736         amfree(qname);
2737         return 2;       /* fatal problem */
2738     }
2739
2740     /* tell the taper to read from a port number of its choice */
2741
2742     taper_cmd(PORT_WRITE, dp, NULL, sched(dp)->level, sched(dp)->datestamp);
2743     cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
2744     if(cmd != PORT) {
2745         printf("driver: did not get PORT from taper for %s:%s\n",
2746                 dp->host->hostname, qname);
2747         fflush(stdout);
2748         amfree(qname);
2749         return 2;       /* fatal problem */
2750     }
2751     /* copy port number */
2752     dumper->output_port = atoi(result_argv[2]);
2753
2754     /* tell the dumper to dump to a port */
2755
2756     dumper_cmd(dumper, PORT_DUMP, dp);
2757     dp->host->start_t = time(NULL) + 15;
2758
2759     /* update statistics & print state */
2760
2761     taper_busy = dumper->busy = 1;
2762     dp->host->inprogress += 1;
2763     dp->inprogress = 1;
2764     sched(dp)->timestamp = time((time_t *)0);
2765     allocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2766     idle_reason = NOT_IDLE;
2767
2768     short_dump_state();
2769
2770     /* wait for result from dumper */
2771
2772     cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
2773
2774     switch(cmd) {
2775     case BOGUS:
2776         /* either eof or garbage from dumper */
2777         log_add(L_WARNING, "%s pid %ld is messed up, ignoring it.\n",
2778                 dumper->name, (long)dumper->pid);
2779         dumper->down = 1;       /* mark it down so it isn't used again */
2780         failed = 1;     /* dump failed, must still finish up with taper */
2781         break;
2782
2783     case DONE: /* DONE <handle> <origsize> <dumpsize> <dumptime> <errstr> */
2784         /* everything went fine */
2785         origsize = (off_t)atof(result_argv[3]);
2786         /*dumpsize = (off_t)atof(result_argv[4]);*/
2787         dumptime = (time_t)atof(result_argv[5]);
2788         break;
2789
2790     case NO_ROOM: /* NO-ROOM <handle> */
2791         dumper_cmd(dumper, ABORT, dp);
2792         cmd = getresult(dumper->fd, 1, &result_argc, result_argv, MAX_ARGS+1);
2793         assert(cmd == ABORT_FINISHED);
2794
2795     case TRYAGAIN: /* TRY-AGAIN <handle> <errstr> */
2796     default:
2797         /* dump failed, but we must still finish up with taper */
2798         /* problem with dump, possibly nonfatal, retry one time */
2799         sched(dp)->attempted++;
2800         failed = sched(dp)->attempted;
2801         dumper_tryagain = 1;
2802         break;
2803
2804     case FAILED: /* FAILED <handle> <errstr> */
2805         /* dump failed, but we must still finish up with taper */
2806         failed = 2;     /* fatal problem with dump */
2807         break;
2808     }
2809
2810     /*
2811      * Note that at this point, even if the dump above failed, it may
2812      * not be a fatal failure if taper below says we can try again.
2813      * E.g. a dumper failure above may actually be the result of a
2814      * tape overflow, which in turn causes dump to see "broken pipe",
2815      * "no space on device", etc., since taper closed the port first.
2816      */
2817
2818     continue_port_dump:
2819
2820     cmd = getresult(taper, 1, &result_argc, result_argv, MAX_ARGS+1);
2821
2822     switch(cmd) {
2823     case PARTIAL:
2824     case DONE: /* DONE <handle> <label> <tape file> <err mess> */
2825         if(result_argc != 5) {
2826             error("error [dump to tape DONE result_argc != 5: %d]", result_argc);
2827             /*NOTREACHED*/
2828         }
2829
2830         if(failed == 1) goto tryagain;  /* dump didn't work */
2831         else if(failed == 2) goto failed_dumper;
2832
2833         free_serial(result_argv[2]);
2834
2835         dumpsize = (off_t)0;
2836         if (*result_argv[5] == '"') {
2837             /* String was quoted */
2838             OFF_T_FMT_TYPE dumpsize_ = (OFF_T_FMT_TYPE)0;
2839             rc = sscanf(result_argv[5],"\"[sec %lf kb " OFF_T_FMT " ",
2840                         &tapetime, &dumpsize_);
2841             dumpsize = (off_t)dumpsize_;
2842         } else {
2843             /* String was not quoted */
2844             OFF_T_FMT_TYPE dumpsize_ = (OFF_T_FMT_TYPE)0;
2845             rc = sscanf(result_argv[5],"[sec %lf kb " OFF_T_FMT " ",
2846                         &tapetime, &dumpsize_);
2847             dumpsize = (off_t)dumpsize_;
2848         }
2849         if (rc < 2) {
2850             error("error [malformed result: %d items matched in '%s']",
2851                   rc, result_argv[5]);
2852             /*NOTREACHED*/
2853         }
2854
2855         if(cmd == DONE) {
2856             /* every thing went fine */
2857             update_info_dumper(dp, origsize, dumpsize, dumptime);
2858             filenum = OFF_T_ATOI(result_argv[4]);
2859             update_info_taper(dp, result_argv[3], filenum, sched(dp)->level);
2860             /* note that update_info_dumper() must be run before
2861                update_info_taper(), since update_info_dumper overwrites
2862                tape information.  */
2863         }
2864
2865         break;
2866
2867     case TRYAGAIN: /* TRY-AGAIN <handle> <err mess> */
2868         tape_left = tape_length;
2869         current_tape++;
2870         if(dumper_tryagain == 0) {
2871             sched(dp)->attempted++;
2872             if(sched(dp)->attempted > failed)
2873                 failed = sched(dp)->attempted;
2874         }
2875     tryagain:
2876         if(failed <= 1)
2877             headqueue_disk(&runq, dp);
2878     failed_dumper:
2879         update_failed_dump_to_tape(dp);
2880         free_serial(result_argv[2]);
2881         break;
2882
2883     case SPLIT_CONTINUE:  /* SPLIT_CONTINUE <handle> <new_label> */
2884         if (result_argc != 3) {
2885             error("error [taper SPLIT_CONTINUE result_argc != 3: %d]", result_argc);
2886             /*NOTREACHED*/
2887         }
2888         fprintf(stderr, "driver: Got SPLIT_CONTINUE %s %s\n",
2889                 result_argv[2], result_argv[3]);
2890         goto continue_port_dump;
2891
2892     case SPLIT_NEEDNEXT:
2893         fprintf(stderr, "driver: Got SPLIT_NEEDNEXT %s %s\n", result_argv[2], result_argv[3]);
2894
2895         goto continue_port_dump;
2896
2897     case TAPE_ERROR: /* TAPE-ERROR <handle> <err mess> */
2898     case BOGUS:
2899     default:
2900         update_failed_dump_to_tape(dp);
2901         free_serial(result_argv[2]);
2902         failed = 2;     /* fatal problem */
2903         start_degraded_mode(&runq);
2904         break;
2905     }
2906
2907     /* reset statistics & return */
2908
2909     taper_busy = dumper->busy = 0;
2910     dp->host->inprogress -= 1;
2911     dp->inprogress = 0;
2912     deallocate_bandwidth(dp->host->netif, sched(dp)->est_kps);
2913     amfree(qname);
2914
2915     return failed;
2916 }
2917
2918 static int
2919 queue_length(
2920     disklist_t  q)
2921 {
2922     disk_t *p;
2923     int len;
2924
2925     for(len = 0, p = q.head; p != NULL; len++, p = p->next)
2926         (void)len;      /* Quiet lint */
2927     return len;
2928 }
2929
2930 static void
2931 short_dump_state(void)
2932 {
2933     int i, nidle;
2934     char *wall_time;
2935
2936     wall_time = walltime_str(curclock());
2937
2938     printf("driver: state time %s ", wall_time);
2939     printf("free kps: %lu space: " OFF_T_FMT " taper: ",
2940            free_kps((interface_t *)0),
2941            (OFF_T_FMT_TYPE)free_space());
2942     if(degraded_mode) printf("DOWN");
2943     else if(!taper_busy) printf("idle");
2944     else printf("writing");
2945     nidle = 0;
2946     for(i = 0; i < inparallel; i++) if(!dmptable[i].busy) nidle++;
2947     printf(" idle-dumpers: %d", nidle);
2948     printf(" qlen tapeq: %d", queue_length(tapeq));
2949     printf(" runq: %d", queue_length(runq));
2950     printf(" roomq: %d", queue_length(roomq));
2951     printf(" wakeup: %d", (int)sleep_time);
2952     printf(" driver-idle: %s\n", idle_strings[idle_reason]);
2953     interface_state(wall_time);
2954     holdingdisk_state(wall_time);
2955     fflush(stdout);
2956 }
2957
2958 #if 0
2959 static void
2960 dump_state(
2961     const char *str)
2962 {
2963     int i;
2964     disk_t *dp;
2965     char *qname;
2966
2967     printf("================\n");
2968     printf("driver state at time %s: %s\n", walltime_str(curclock()), str);
2969     printf("free kps: %lu, space: " OFF_T_FMT "\n",
2970            free_kps((interface_t *)0),
2971            (OFF_T_FMT_TYPE)free_space());
2972     if(degraded_mode) printf("taper: DOWN\n");
2973     else if(!taper_busy) printf("taper: idle\n");
2974     else printf("taper: writing %s:%s.%d est size " OFF_T_FMT "\n",
2975                 taper_disk->host->hostname, taper_disk->name,
2976                 sched(taper_disk)->level,
2977                 sched(taper_disk)->est_size);
2978     for(i = 0; i < inparallel; i++) {
2979         dp = dmptable[i].dp;
2980         if(!dmptable[i].busy)
2981           printf("%s: idle\n", dmptable[i].name);
2982         else
2983           qname = quote_string(dp->name);
2984           printf("%s: dumping %s:%s.%d est kps %d size " OFF_T_FMT " time %lu\n",
2985                 dmptable[i].name, dp->host->hostname, qname, sched(dp)->level,
2986                 sched(dp)->est_kps, sched(dp)->est_size, sched(dp)->est_time);
2987           amfree(qname);
2988     }
2989     dump_queue("TAPE", tapeq, 5, stdout);
2990     dump_queue("ROOM", roomq, 5, stdout);
2991     dump_queue("RUN ", runq, 5, stdout);
2992     printf("================\n");
2993     fflush(stdout);
2994 }
2995 #endif