X-Git-Url: https://git.gag.com/?a=blobdiff_plain;f=server-src%2Fplanner.c;h=f7a38be691eeedbdf900a7043fc8dc3e9fbde377;hb=HEAD;hp=118d54e8b232956f3682661acedf6bf0a4e6c211;hpb=2627875b7d18858bc1f9f7652811e4d8c15a23eb;p=debian%2Famanda diff --git a/server-src/planner.c b/server-src/planner.c index 118d54e..f7a38be 100644 --- a/server-src/planner.c +++ b/server-src/planner.c @@ -1,6 +1,7 @@ /* * Amanda, The Advanced Maryland Automatic Network Disk Archiver * Copyright (c) 1991-1999 University of Maryland at College Park + * Copyright (c) 2007-2012 Zmanda, Inc. All Rights Reserved. * All Rights Reserved. * * Permission to use, copy, modify, distribute, and sell this software and its @@ -30,6 +31,7 @@ */ #include "amanda.h" #include "arglist.h" +#include "find.h" #include "conffile.h" #include "diskfile.h" #include "tapefile.h" @@ -81,16 +83,22 @@ int conf_usetimestamps; #define DISK_PARTIALY_DONE 2 #define DISK_DONE 3 +typedef struct one_est_s { + int level; + gint64 nsize; /* native size */ + gint64 csize; /* compressed size */ + char *dumpdate; + int guessed; /* If server guessed the estimate size */ +} one_est_t; +static one_est_t default_one_est = {-1, -1, -1, "INVALID_DATE", 0}; + typedef struct est_s { int state; int got_estimate; int dump_priority; - int dump_level; - gint64 dump_nsize; /* native size */ - gint64 dump_csize; /* compressed size */ - int degr_level; /* if dump_level == 0, what would be the inc level */ - gint64 degr_nsize; /* native degraded size */ - gint64 degr_csize; /* compressed degraded size */ + one_est_t *dump_est; + one_est_t *degr_est; + one_est_t estimate[MAX_LEVELS]; int last_level; gint64 last_lev0size; int next_level0; @@ -100,10 +108,8 @@ typedef struct est_s { double fullrate, incrrate; double fullcomp, incrcomp; char *errstr; - int level[MAX_LEVELS]; - char *dumpdate[MAX_LEVELS]; - gint64 est_size[MAX_LEVELS]; char *degr_mesg; + info_t *info; } est_t; #define est(dp) ((est_t *)(dp)->up) @@ -161,6 +167,7 @@ static void delay_dumps(void); static int promote_highest_priority_incremental(void); static int promote_hills(void); static void output_scheduleline(disk_t *dp); +static void server_estimate(disk_t *dp, int i, info_t *info, int level); int main(int, char **); int @@ -181,10 +188,18 @@ main( char *qname; int nb_disk; char *errstr = NULL; - config_overwrites_t *cfg_ovr = NULL; + config_overrides_t *cfg_ovr = NULL; char *cfg_opt = NULL; int planner_setuid; int exit_status = EXIT_SUCCESS; + gboolean no_taper = FALSE; + gboolean from_client = FALSE; + gboolean exact_match = FALSE; + + if (argc > 1 && argv && argv[1] && g_str_equal(argv[1], "--version")) { + printf("planner-%s\n", VERSION); + return (0); + } /* * Configure program for internationalization: @@ -204,24 +219,25 @@ main( dbopen(DBG_SUBDIR_SERVER); - cfg_ovr = extract_commandline_config_overwrites(&argc, &argv); + cfg_ovr = extract_commandline_config_overrides(&argc, &argv); if (argc > 1) cfg_opt = argv[1]; + set_config_overrides(cfg_ovr); config_init(CONFIG_INIT_EXPLICIT_NAME | CONFIG_INIT_USE_CWD, cfg_opt); - apply_config_overwrites(cfg_ovr); /* conf_diskfile is freed later, as it may be used in an error message */ conf_diskfile = config_dir_relative(getconf_str(CNF_DISKFILE)); read_diskfile(conf_diskfile, &origq); + disable_skip_disk(&origq); /* Don't die when child closes pipe */ signal(SIGPIPE, SIG_IGN); setvbuf(stderr, (char *)NULL, (int)_IOLBF, 0); - erroutput_type = (ERR_AMANDALOG|ERR_INTERACTIVE); - set_logerror(logerror); + add_amanda_log_handler(amanda_log_stderr); + add_amanda_log_handler(amanda_log_trace_log); if (!planner_setuid) { error(_("planner must be run setuid root")); @@ -233,7 +249,7 @@ main( safe_cd(); - check_running_as(RUNNING_AS_DUMPUSER); + check_running_as(RUNNING_AS_ROOT | RUNNING_AS_UID_ONLY); dbrename(get_config_name(), DBG_SUBDIR_SERVER); @@ -245,16 +261,31 @@ main( log_add(L_INFO, "%s pid %ld", get_pname(), (long)getpid()); g_fprintf(stderr, _("%s: pid %ld executable %s version %s\n"), - get_pname(), (long) getpid(), argv[0], version()); + get_pname(), (long) getpid(), argv[0], VERSION); for (i = 0; version_info[i] != NULL; i++) g_fprintf(stderr, _("%s: %s"), get_pname(), version_info[i]); diskarg_offset = 2; - if (argc > 3 && strcmp(argv[2], "--starttime") == 0) { - planner_timestamp = stralloc(argv[3]); + if (argc - diskarg_offset > 1 && strcmp(argv[diskarg_offset], "--starttime") == 0) { + planner_timestamp = stralloc(argv[diskarg_offset+1]); diskarg_offset += 2; } + if (argc - diskarg_offset > 0 && strcmp(argv[diskarg_offset], "--no-taper") == 0) { + no_taper = TRUE; + diskarg_offset += 1; + } + if (argc - diskarg_offset > 0 && strcmp(argv[diskarg_offset], "--from-client") == 0) { + from_client = TRUE; + diskarg_offset += 1; + } + if (argc - diskarg_offset > 0 && g_str_equal(argv[diskarg_offset], + "--exact_match")) { + exact_match = TRUE; + diskarg_offset += 1; + } + + run_server_global_scripts(EXECUTE_ON_PRE_ESTIMATE, get_config_name()); /* * 1. Networking Setup @@ -276,27 +307,6 @@ main( /*NOTREACHED*/ } - errstr = match_disklist(&origq, argc-diskarg_offset, - argv+diskarg_offset); - if (errstr) { - g_fprintf(stderr,"%s",errstr); - amfree(errstr); - exit_status = EXIT_FAILURE; - } - nb_disk = 0; - for(dp = origq.head; dp != NULL; dp = dp->next) { - if(dp->todo) { - qname = quote_string(dp->name); - log_add(L_DISK, "%s %s", dp->host->hostname, qname); - amfree(qname); - nb_disk++; - } - } - - if(nb_disk == 0) { - error(_("no DLE to backup")); - /*NOTREACHED*/ - } amfree(conf_diskfile); conf_tapelist = config_dir_relative(getconf_str(CNF_TAPELIST)); @@ -325,7 +335,7 @@ main( conf_tapecycle = getconf_int(CNF_TAPECYCLE); conf_etimeout = (time_t)getconf_int(CNF_ETIMEOUT); conf_reserve = getconf_int(CNF_RESERVE); - conf_autoflush = getconf_boolean(CNF_AUTOFLUSH); + conf_autoflush = getconf_no_yes_all(CNF_AUTOFLUSH); conf_usetimestamps = getconf_boolean(CNF_USETIMESTAMPS); today = time(0); @@ -345,6 +355,47 @@ main( g_fprintf(stderr, _("%s: timestamp %s\n"), get_pname(), planner_timestamp); + errstr = match_disklist(&origq, exact_match, argc-diskarg_offset, + argv+diskarg_offset); + if (errstr) { + g_fprintf(stderr,"%s",errstr); + exit_status = EXIT_FAILURE; + } + + for (dp = origq.head; dp != NULL; dp = dp->next) { + if (dp->todo) { + if (from_client) { + if (!dp->dump_limit || !dp->dump_limit->same_host) + dp->todo = 0; + } else { + if (dp->dump_limit && !dp->dump_limit->server) + dp->todo = 0; + } + } + } + + nb_disk = 0; + for (dp = origq.head; dp != NULL; dp = dp->next) { + if (dp->todo) { + qname = quote_string(dp->name); + log_add(L_DISK, "%s %s", dp->host->hostname, qname); + amfree(qname); + nb_disk++; + } + } + + if (nb_disk == 0) { + if (errstr) { + error(_("no DLE to backup; %s"), errstr); + } else { + error(_("no DLE to backup")); + } + /*NOTREACHED*/ + } else if (errstr) { + log_add(L_WARNING, "WARNING: %s", errstr); + } + amfree(errstr); + /* some initializations */ if(conf_runspercycle == 0) { @@ -369,9 +420,15 @@ main( tape = lookup_tapetype(conf_tapetype); if(conf_maxdumpsize > (gint64)0) { tape_length = conf_maxdumpsize; + g_fprintf(stderr, "planner: tape_length is set from maxdumpsize (%jd KB)\n", + (intmax_t)conf_maxdumpsize); } else { tape_length = tapetype_get_length(tape) * (gint64)conf_runtapes; + g_fprintf(stderr, "planner: tape_length is set from tape length (%jd KB) * runtapes (%d) == %jd KB\n", + (intmax_t)tapetype_get_length(tape), + conf_runtapes, + (intmax_t)tape_length); } tape_mark = (size_t)tapetype_get_filemark(tape); tt_blocksize_kb = (size_t)tapetype_get_blocksize(tape); @@ -391,12 +448,13 @@ main( g_fprintf(stderr,_("\nSENDING FLUSHES...\n")); - if(conf_autoflush) { + if(conf_autoflush && !no_taper) { dumpfile_t file; GSList *holding_list, *holding_file; char *qdisk, *qhname; - /* get *all* flushable files in holding */ + /* get *all* flushable files in holding, without checking against + * the disklist (which may not contain some of the dumps) */ holding_list = holding_get_files_for_flush(NULL); for(holding_file=holding_list; holding_file != NULL; holding_file = holding_file->next) { @@ -410,6 +468,13 @@ main( continue; } + /* see if this matches the command-line arguments */ + if (conf_autoflush == 1 && + !match_dumpfile(&file, exact_match, argc-diskarg_offset, + argv+diskarg_offset)) { + continue; + } + qdisk = quote_string(file.disk); qhname = quote_string((char *)holding_file->data); log_add(L_DISK, "%s %s", file.name, qdisk); @@ -431,7 +496,7 @@ main( amfree(qhname); dumpfile_free_data(&file); } - g_slist_free_full(holding_list); + slist_free_full(holding_list, g_free); holding_list = NULL; } g_fprintf(stderr, _("ENDFLUSH\n")); @@ -521,6 +586,8 @@ main( while(!empty(estq)) analyze_estimate(dequeue_disk(&estq)); while(!empty(failq)) handle_failed(dequeue_disk(&failq)); + run_server_global_scripts(EXECUTE_ON_POST_ESTIMATE, get_config_name()); + /* * At this point, all the disks are on schedq sorted by priority. * The total estimated size of the backups is in total_size. @@ -535,9 +602,9 @@ main( qname = quote_string(dp->name); g_fprintf(stderr, _(" %s %s pri %d lev %d nsize %lld csize %lld\n"), dp->host->hostname, qname, est(dp)->dump_priority, - est(dp)->dump_level, - (long long)est(dp)->dump_nsize, - (long long)est(dp)->dump_csize); + est(dp)->dump_est->level, + (long long)est(dp)->dump_est->nsize, + (long long)est(dp)->dump_est->csize); amfree(qname); } } @@ -647,7 +714,8 @@ main( static void askfor(est_t *, int, int, info_t *); static int last_level(info_t *info); /* subroutines */ -static gint64 est_size(disk_t *dp, int level); +static one_est_t *est_for_level(disk_t *dp, int level); +static void est_csize(disk_t *dp, one_est_t *one_est); static gint64 est_tape_size(disk_t *dp, int level); static int next_level0(disk_t *dp, info_t *info); static int runs_at(info_t *info, int lev); @@ -672,17 +740,21 @@ static void askfor( } if (lev == -1) { - ep->level[seq] = -1; - ep->dumpdate[seq] = (char *)0; - ep->est_size[seq] = (gint64)-2; + ep->estimate[seq].level = -1; + ep->estimate[seq].dumpdate = (char *)0; + ep->estimate[seq].nsize = (gint64)-3; + ep->estimate[seq].csize = (gint64)-3; + ep->estimate[seq].guessed = 0; return; } - ep->level[seq] = lev; + ep->estimate[seq].level = lev; - ep->dumpdate[seq] = stralloc(get_dumpdate(info,lev)); + ep->estimate[seq].dumpdate = stralloc(get_dumpdate(info,lev)); - ep->est_size[seq] = (gint64)-2; + ep->estimate[seq].nsize = (gint64)-3; + ep->estimate[seq].csize = (gint64)-3; + ep->estimate[seq].guessed = 0; return; } @@ -692,7 +764,7 @@ setup_estimate( disk_t *dp) { est_t *ep; - info_t info; + info_t *info; int i; char *qname; int overwrite_runs; @@ -706,27 +778,60 @@ setup_estimate( /* get current information about disk */ - if(get_info(dp->host->hostname, dp->name, &info)) { + info = g_new0(info_t, 1); + if(get_info(dp->host->hostname, dp->name, info)) { /* no record for this disk, make a note of it */ - log_add(L_INFO, _("Adding new disk %s:%s."), dp->host->hostname, dp->name); + log_add(L_INFO, _("Adding new disk %s:%s."), dp->host->hostname, qname); + } + + if (dp->data_path == DATA_PATH_DIRECTTCP) { + if (dp->compress != COMP_NONE) { + log_add(L_FAIL, _("%s %s %s 0 [Can't compress directtcp data-path]"), + dp->host->hostname, qname, planner_timestamp); + g_fprintf(stderr,_("%s:%s lev 0 skipped can't compress directtcp data-path\n"), + dp->host->hostname, qname); + amfree(qname); + return; + } + if (dp->encrypt != ENCRYPT_NONE) { + log_add(L_FAIL, _("%s %s %s 0 [Can't encrypt directtcp data-path]"), + dp->host->hostname, qname, planner_timestamp); + g_fprintf(stderr,_("%s:%s lev 0 skipped can't encrypt directtcp data-path\n"), + dp->host->hostname, qname); + amfree(qname); + return; + } + if (dp->to_holdingdisk == HOLD_REQUIRED) { + log_add(L_FAIL, _("%s %s %s 0 [Holding disk can't be use for directtcp data-path]"), + dp->host->hostname, qname, planner_timestamp); + g_fprintf(stderr,_("%s:%s lev 0 skipped Holding disk can't be use for directtcp data-path\n"), + dp->host->hostname, qname); + amfree(qname); + return; + } else if (dp->to_holdingdisk == HOLD_AUTO) { + g_fprintf(stderr,_("%s:%s Disabling holding disk\n"), + dp->host->hostname, qname); + dp->to_holdingdisk = HOLD_NEVER; + } } /* setup working data struct for disk */ ep = alloc(SIZEOF(est_t)); dp->up = (void *) ep; + ep->info = info; ep->state = DISK_READY; - ep->dump_nsize = (gint64)-1; - ep->dump_csize = (gint64)-1; ep->dump_priority = dp->priority; ep->errstr = 0; ep->promote = 0; ep->post_dle = 0; ep->degr_mesg = NULL; + ep->dump_est = &default_one_est; + ep->degr_est = &default_one_est; /* calculated fields */ - if (ISSET(info.command, FORCE_FULL)) { + if (ISSET(info->command, FORCE_FULL)) { /* force a level 0, kind of like a new disk */ if(dp->strategy == DS_NOFULL) { /* @@ -740,22 +845,25 @@ setup_estimate( * hosed when that tape gets re-used next. Disallow this for * now. */ - log_add(L_ERROR, + log_add(L_WARNING, _("Cannot force full dump of %s:%s with no-full option."), dp->host->hostname, qname); /* clear force command */ - CLR(info.command, FORCE_FULL); - if(put_info(dp->host->hostname, dp->name, &info)) { - error(_("could not put info record for %s:%s: %s"), - dp->host->hostname, qname, strerror(errno)); - /*NOTREACHED*/ - } - ep->last_level = last_level(&info); - ep->next_level0 = next_level0(dp, &info); - } - else { - ep->degr_mesg = _("Can't switch to degraded mode when using a force-full disk"); + CLR(info->command, FORCE_FULL); + ep->last_level = last_level(info); + ep->next_level0 = next_level0(dp, info); + } else if (dp->strategy == DS_INCRONLY) { + log_add(L_WARNING, + _("Cannot force full dump of %s:%s with incronly option."), + dp->host->hostname, qname); + + /* clear force command */ + CLR(info->command, FORCE_FULL); + ep->last_level = last_level(info); + ep->next_level0 = next_level0(dp, info); + } else { + ep->degr_mesg = _("Skipping: force-full disk can't be dumped in degraded mode"); ep->last_level = -1; ep->next_level0 = -conf_dumpcycle; log_add(L_INFO, _("Forcing full dump of %s:%s as directed."), @@ -765,44 +873,44 @@ setup_estimate( else if(dp->strategy == DS_NOFULL) { /* force estimate of level 1 */ ep->last_level = 1; - ep->next_level0 = next_level0(dp, &info); + ep->next_level0 = next_level0(dp, info); } else { - ep->last_level = last_level(&info); - ep->next_level0 = next_level0(dp, &info); + ep->last_level = last_level(info); + ep->next_level0 = next_level0(dp, info); } /* adjust priority levels */ /* warn if dump will be overwritten */ - if (ep->last_level > -1 && strlen(info.inf[0].label) > 0) { - overwrite_runs = when_overwrite(info.inf[0].label); + if (ep->last_level > -1 && strlen(info->inf[0].label) > 0) { + overwrite_runs = when_overwrite(info->inf[0].label); if(overwrite_runs == 0) { log_add(L_WARNING, _("Last full dump of %s:%s " "on tape %s overwritten on this run."), - dp->host->hostname, qname, info.inf[0].label); + dp->host->hostname, qname, info->inf[0].label); } else if(overwrite_runs <= RUNS_REDZONE) { log_add(L_WARNING, plural(_("Last full dump of %s:%s on tape %s overwritten in %d run."), _("Last full dump of %s:%s on tape %s overwritten in %d runs."), overwrite_runs), - dp->host->hostname, qname, info.inf[0].label, + dp->host->hostname, qname, info->inf[0].label, overwrite_runs); } } /* warn if last level 1 will be overwritten */ - if (ep->last_level > 1 && strlen(info.inf[1].label) > 0) { - overwrite_runs = when_overwrite(info.inf[1].label); + if (ep->last_level > 1 && strlen(info->inf[1].label) > 0) { + overwrite_runs = when_overwrite(info->inf[1].label); if(overwrite_runs == 0) { log_add(L_WARNING, _("Last level 1 dump of %s:%s " "on tape %s overwritten on this run, resetting to level 1"), - dp->host->hostname, qname, info.inf[1].label); + dp->host->hostname, qname, info->inf[1].label); ep->last_level = 0; } else if(overwrite_runs <= RUNS_REDZONE) { log_add(L_WARNING, plural(_("Last level 1 dump of %s:%s on tape %s overwritten in %d run."), _("Last level 1 dump of %s:%s on tape %s overwritten in %d runs."), overwrite_runs), - dp->host->hostname, qname, info.inf[1].label, + dp->host->hostname, qname, info->inf[1].label, overwrite_runs); } } @@ -814,7 +922,7 @@ setup_estimate( dp->host->hostname, qname, (-ep->next_level0)); ep->dump_priority -= ep->next_level0; } - else if (ISSET(info.command, FORCE_FULL)) + else if (ISSET(info->command, FORCE_FULL)) ep->dump_priority += 1; /* else XXX bump up the priority of incrementals that failed last night */ @@ -823,11 +931,11 @@ setup_estimate( if(dp->skip_full && dp->strategy != DS_NOINC) { if(ep->next_level0 <= 0) { /* update the date field */ - info.inf[0].date = today; - CLR(info.command, FORCE_FULL); + info->inf[0].date = today; + CLR(info->command, FORCE_FULL); ep->next_level0 += conf_dumpcycle; ep->last_level = 0; - if(put_info(dp->host->hostname, dp->name, &info)) { + if(put_info(dp->host->hostname, dp->name, info)) { error(_("could not put info record for %s:%s: %s"), dp->host->hostname, qname, strerror(errno)); /*NOTREACHED*/ @@ -837,9 +945,9 @@ setup_estimate( g_fprintf(stderr,_("%s:%s lev 0 skipped due to skip-full flag\n"), dp->host->hostname, qname); /* don't enqueue the disk */ - askfor(ep, 0, -1, &info); - askfor(ep, 1, -1, &info); - askfor(ep, 2, -1, &info); + askfor(ep, 0, -1, info); + askfor(ep, 1, -1, info); + askfor(ep, 2, -1, info); g_fprintf(stderr, _("%s: SKIPPED %s %s 0 [skip-full]\n"), get_pname(), dp->host->hostname, qname); log_add(L_SUCCESS, _("%s %s %s 0 [skipped: skip-full]"), @@ -859,11 +967,11 @@ setup_estimate( } } - if(dp->strategy == DS_INCRONLY && ep->last_level == -1 && !ISSET(info.command, FORCE_FULL)) { + if(dp->strategy == DS_INCRONLY && ep->last_level == -1 && !ISSET(info->command, FORCE_FULL)) { /* don't enqueue the disk */ - askfor(ep, 0, -1, &info); - askfor(ep, 1, -1, &info); - askfor(ep, 2, -1, &info); + askfor(ep, 0, -1, info); + askfor(ep, 1, -1, info); + askfor(ep, 2, -1, info); log_add(L_FAIL, _("%s %s 19000101 1 [Skipping incronly because no full dump were done]"), dp->host->hostname, qname); g_fprintf(stderr,_("%s:%s lev 1 skipped due to strategy incronly and no full dump were done\n"), @@ -878,9 +986,9 @@ setup_estimate( g_fprintf(stderr,_("%s:%s lev 1 skipped due to skip-incr flag\n"), dp->host->hostname, qname); /* don't enqueue the disk */ - askfor(ep, 0, -1, &info); - askfor(ep, 1, -1, &info); - askfor(ep, 2, -1, &info); + askfor(ep, 0, -1, info); + askfor(ep, 1, -1, info); + askfor(ep, 2, -1, info); g_fprintf(stderr, _("%s: SKIPPED %s %s 1 [skip-incr]\n"), get_pname(), dp->host->hostname, qname); @@ -900,15 +1008,16 @@ setup_estimate( ep->next_level0 = 0; } - if(ep->last_level == 0) ep->level_days = 0; - else ep->level_days = runs_at(&info, ep->last_level); - ep->last_lev0size = info.inf[0].csize; + //if(ep->last_level == 0) ep->level_days = 0; + //else ep->level_days = runs_at(info, ep->last_level); + ep->level_days = runs_at(info, ep->last_level); + ep->last_lev0size = info->inf[0].csize; - ep->fullrate = perf_average(info.full.rate, 0.0); - ep->incrrate = perf_average(info.incr.rate, 0.0); + ep->fullrate = perf_average(info->full.rate, 0.0); + ep->incrrate = perf_average(info->incr.rate, 0.0); - ep->fullcomp = perf_average(info.full.comp, dp->comprate[0]); - ep->incrcomp = perf_average(info.incr.comp, dp->comprate[1]); + ep->fullcomp = perf_average(info->full.comp, dp->comprate[0]); + ep->incrcomp = perf_average(info->incr.comp, dp->comprate[1]); /* determine which estimates to get */ @@ -916,10 +1025,10 @@ setup_estimate( if (dp->strategy == DS_NOINC || (!dp->skip_full && - (!ISSET(info.command, FORCE_BUMP) || + (!ISSET(info->command, FORCE_BUMP) || dp->skip_incr || ep->last_level == -1))) { - if(info.command & FORCE_BUMP && ep->last_level == -1) { + if(ISSET(info->command, FORCE_BUMP) && ep->last_level == -1) { log_add(L_INFO, _("Remove force-bump command of %s:%s because it's a new disk."), dp->host->hostname, qname); @@ -927,13 +1036,17 @@ setup_estimate( switch (dp->strategy) { case DS_STANDARD: case DS_NOINC: - askfor(ep, i++, 0, &info); + askfor(ep, i++, 0, info); + if (ep->last_level == -1) + ep->degr_mesg = _("Skipping: new disk can't be dumped in degraded mode"); + else + ep->degr_mesg = _("Skipping: strategy NOINC can't be dumped in degraded mode"); if(dp->skip_full) { - log_add(L_INFO, _("Ignoring skip_full for %s:%s " + log_add(L_INFO, _("Ignoring skip-full for %s:%s " "because the strategy is NOINC."), dp->host->hostname, qname); } - if(info.command & FORCE_BUMP) { + if(ISSET(info->command, FORCE_BUMP)) { log_add(L_INFO, _("Ignoring FORCE_BUMP for %s:%s because the strategy is NOINC."), dp->host->hostname, qname); @@ -945,7 +1058,7 @@ setup_estimate( break; case DS_INCRONLY: - if (ISSET(info.command, FORCE_FULL)) + if (ISSET(info->command, FORCE_FULL)) ep->last_level = 0; break; } @@ -954,9 +1067,9 @@ setup_estimate( if(!dp->skip_incr && !(dp->strategy == DS_NOINC)) { if(ep->last_level == -1) { /* a new disk */ if (ep->degr_mesg == NULL) - ep->degr_mesg = _("Can't switch to degraded mode when using a new disk"); + ep->degr_mesg = _("Skipping: new disk can't be dumped in degraded mode"); if(dp->strategy == DS_NOFULL || dp->strategy == DS_INCRONLY) { - askfor(ep, i++, 1, &info); + askfor(ep, i++, 1, info); } else { assert(!dp->skip_full); /* should be handled above */ } @@ -965,23 +1078,23 @@ setup_estimate( curr_level = ep->last_level; - if (ISSET(info.command, FORCE_NO_BUMP)) { + if (ISSET(info->command, FORCE_NO_BUMP)) { if(curr_level > 0) { /* level 0 already asked for */ - askfor(ep, i++, curr_level, &info); + askfor(ep, i++, curr_level, info); } log_add(L_INFO,_("Preventing bump of %s:%s as directed."), dp->host->hostname, qname); - ep->degr_mesg = _("Can't switch to degraded mode when using a force-no-bump disk"); - } else if (ISSET(info.command, FORCE_BUMP) + ep->degr_mesg = _("Skipping: force-no-bump disk can't be dumped in degraded mode"); + } else if (ISSET(info->command, FORCE_BUMP) && curr_level + 1 < DUMP_LEVELS) { - askfor(ep, i++, curr_level+1, &info); + askfor(ep, i++, curr_level+1, info); log_add(L_INFO,_("Bumping of %s:%s at level %d as directed."), dp->host->hostname, qname, curr_level+1); - ep->degr_mesg = _("Can't switch to degraded mode when using a force-bump disk"); + ep->degr_mesg = _("Skipping: force-bump disk can't be dumped in degraded mode"); } else if (curr_level == 0) { - askfor(ep, i++, 1, &info); + askfor(ep, i++, 1, info); } else { - askfor(ep, i++, curr_level, &info); + askfor(ep, i++, curr_level, info); /* * If last time we dumped less than the threshold, then this * time we will too, OR the extra size will be charged to both @@ -989,35 +1102,35 @@ setup_estimate( * if we haven't been at this level 2 days, or the dump failed * last night, we can't bump. */ - if((info.inf[curr_level].size == (gint64)0 || /* no data, try it anyway */ - (((info.inf[curr_level].size > bump_thresh(curr_level, info.inf[0].size,dp->bumppercent, dp->bumpsize, dp->bumpmult))) + if((info->inf[curr_level].size == (gint64)0 || /* no data, try it anyway */ + (((info->inf[curr_level].size > bump_thresh(curr_level, info->inf[0].size,dp->bumppercent, dp->bumpsize, dp->bumpmult))) && ep->level_days >= dp->bumpdays)) && curr_level + 1 < DUMP_LEVELS) { - askfor(ep, i++, curr_level+1, &info); + askfor(ep, i++, curr_level+1, info); } - } + } } } - while(i < MAX_LEVELS) /* mark end of estimates */ - askfor(ep, i++, -1, &info); + while(i < MAX_LEVELS) /* mark end of estimates */ + askfor(ep, i++, -1, info); /* debug output */ g_fprintf(stderr, _("setup_estimate: %s:%s: command %u, options: %s " - "last_level %d next_level0 %d level_days %d getting estimates " + "last_level %d next_level0 %d level_days %d getting estimates " "%d (%lld) %d (%lld) %d (%lld)\n"), - dp->host->hostname, qname, info.command, + dp->host->hostname, qname, info->command, dp->strategy == DS_NOFULL ? "no-full" : dp->strategy == DS_INCRONLY ? "incr-only" : dp->skip_full ? "skip-full" : dp->skip_incr ? "skip-incr" : "none", ep->last_level, ep->next_level0, ep->level_days, - ep->level[0], (long long)ep->est_size[0], - ep->level[1], (long long)ep->est_size[1], - ep->level[2], (long long)ep->est_size[2]); + ep->estimate[0].level, (long long)ep->estimate[0].nsize, + ep->estimate[1].level, (long long)ep->estimate[1].nsize, + ep->estimate[2].level, (long long)ep->estimate[2].nsize); - assert(ep->level[0] != -1); + assert(ep->estimate[0].level != -1); enqueue_disk(&startq, dp); amfree(qname); } @@ -1042,35 +1155,42 @@ static int when_overwrite( } /* Return the estimated size for a particular dump */ -static gint64 est_size( +static one_est_t * +est_for_level( disk_t *dp, int level) { int i; - for(i = 0; i < MAX_LEVELS; i++) { - if(level == est(dp)->level[i]) - return est(dp)->est_size[i]; + if (level < 0 || level >= DUMP_LEVELS) + return &default_one_est; + + for (i = 0; i < MAX_LEVELS; i++) { + if (level == est(dp)->estimate[i].level) { + if (est(dp)->estimate[i].csize <= -1) { + est_csize(dp, &est(dp)->estimate[i]); + } + return &est(dp)->estimate[i]; + } } - return (gint64)-1; + return &default_one_est; } /* Return the estimated on-tape size of a particular dump */ -static gint64 est_tape_size( - disk_t *dp, - int level) +static void +est_csize( + disk_t *dp, + one_est_t *one_est) { - gint64 size; + gint64 size = one_est->nsize; double ratio; - size = est_size(dp, level); - - if(size == (gint64)-1) return size; - - if(dp->compress == COMP_NONE) - return size; + if (dp->compress == COMP_NONE) { + one_est->csize = one_est->nsize; + return; + } - if(level == 0) ratio = est(dp)->fullcomp; + if (one_est->level == 0) ratio = est(dp)->fullcomp; else ratio = est(dp)->incrcomp; /* @@ -1082,7 +1202,7 @@ static gint64 est_tape_size( * (RUG@USM.Uni-Muenchen.DE) */ - if(ratio > 1.1) ratio = 1.1; + if (ratio > 1.1) ratio = 1.1; size = (gint64)((double)size * ratio); @@ -1091,11 +1211,23 @@ static gint64 est_tape_size( * size goes back greater than zero. It may not be right, but * indicates we did get an estimate. */ - if(size <= (gint64)0) { + if (size <= (gint64)0) { size = (gint64)1; } - return size; + one_est->csize = size; +} + +static gint64 est_tape_size( + disk_t *dp, + int level) +{ + one_est_t *dump_est; + + dump_est = est_for_level(dp, level); + if (dump_est->level >= 0 && dump_est->csize <= -1) + est_csize(dp, dump_est); + return dump_est->csize; } @@ -1166,10 +1298,9 @@ static int runs_at( last = last_level(info); if(lev != last) return 0; - if(lev == 0) return 1; - if(info->consecutive_runs != -1) return info->consecutive_runs; + if(lev == 0) return 1; /* to keep compatibility with old infofile */ cur_tape = lookup_tapelabel(info->inf[lev].label); @@ -1232,17 +1363,13 @@ static void get_estimates(void) hostp = dp->host; if(hostp->up == HOST_READY) { something_started = 1; + run_server_host_scripts(EXECUTE_ON_PRE_HOST_ESTIMATE, + get_config_name(), hostp); for(dp1 = hostp->disks; dp1 != NULL; dp1 = dp1->hostnext) { if (dp1->todo) - run_server_scripts(EXECUTE_ON_PRE_HOST_ESTIMATE, + run_server_dle_scripts(EXECUTE_ON_PRE_DLE_ESTIMATE, get_config_name(), dp1, - est(dp1)->level[0]); - } - for(dp1 = hostp->disks; dp1 != NULL; dp1 = dp1->hostnext) { - if (dp1->todo) - run_server_scripts(EXECUTE_ON_PRE_DLE_ESTIMATE, - get_config_name(), dp1, - est(dp1)->level[0]); + est(dp1)->estimate[0].level); } getsize(hostp); protocol_check(); @@ -1265,51 +1392,26 @@ static void get_estimates(void) while(!empty(pestq)) { disk_t *dp = dequeue_disk(&pestq); char * qname = quote_string(dp->name); - - if(est(dp)->level[0] != -1 && est(dp)->est_size[0] < (gint64)0) { - if(est(dp)->est_size[0] == (gint64)-1) { - log_add(L_WARNING, _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, est(dp)->level[0]); - } - else { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d timed out."), - dp->host->hostname, qname, est(dp)->level[0]); - } - est(dp)->level[0] = -1; - } - - if(est(dp)->level[1] != -1 && est(dp)->est_size[1] < (gint64)0) { - if(est(dp)->est_size[1] == (gint64)-1) { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, est(dp)->level[1]); - } - else { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d timed out."), - dp->host->hostname, qname, est(dp)->level[1]); - } - est(dp)->level[1] = -1; - } - - if(est(dp)->level[2] != -1 && est(dp)->est_size[2] < (gint64)0) { - if(est(dp)->est_size[2] == (gint64)-1) { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, est(dp)->level[2]); - } - else { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d timed out."), - dp->host->hostname, qname, est(dp)->level[2]); - } - est(dp)->level[2] = -1; - } - - if((est(dp)->level[0] != -1 && est(dp)->est_size[0] > (gint64)0) || - (est(dp)->level[1] != -1 && est(dp)->est_size[1] > (gint64)0) || - (est(dp)->level[2] != -1 && est(dp)->est_size[2] > (gint64)0)) { + int i; + + for (i=0; i < MAX_LEVELS; i++) { + if (est(dp)->estimate[i].level != -1 && + est(dp)->estimate[i].nsize < (gint64)0) { + if (est(dp)->estimate[i].nsize == (gint64)-3) { + log_add(L_WARNING, + _("disk %s:%s, estimate of level %d timed out."), + dp->host->hostname, qname, est(dp)->estimate[i].level); + } + est(dp)->estimate[i].level = -1; + } + } + + if ((est(dp)->estimate[0].level != -1 && + est(dp)->estimate[0].nsize > (gint64)0) || + (est(dp)->estimate[1].level != -1 && + est(dp)->estimate[1].nsize > (gint64)0) || + (est(dp)->estimate[2].level != -1 && + est(dp)->estimate[2].nsize > (gint64)0)) { enqueue_disk(&estq, dp); } else { @@ -1333,6 +1435,8 @@ static void getsize( char * calcsize; char * qname, *b64disk = NULL; char * qdevice, *b64device = NULL; + estimate_t estimate; + estimatelist_t el; assert(hostp->disks != NULL); @@ -1382,25 +1486,83 @@ static void getsize( estimates = 0; for(dp = hostp->disks; dp != NULL; dp = dp->hostnext) { char *s = NULL; + char *es; size_t s_len = 0; + GPtrArray *errarray; if(dp->todo == 0) continue; if(est(dp)->state != DISK_READY) continue; est(dp)->got_estimate = 0; - if(est(dp)->level[0] == -1) { + if (est(dp)->estimate[0].level == -1) { est(dp)->state = DISK_DONE; continue; } qname = quote_string(dp->name); + + errarray = validate_optionstr(dp); + if (errarray->len > 0) { + guint i; + for (i=0; i < errarray->len; i++) { + log_add(L_FAIL, _("%s %s %s 0 [%s]"), + dp->host->hostname, qname, + planner_timestamp, + (char *)g_ptr_array_index(errarray, i)); + } + amfree(qname); + est(dp)->state = DISK_DONE; + continue; + } + b64disk = amxml_format_tag("disk", dp->name); qdevice = quote_string(dp->device); + estimate = (estimate_t)GPOINTER_TO_INT(dp->estimatelist->data); if (dp->device) b64device = amxml_format_tag("diskdevice", dp->device); - if (dp->estimate == ES_CLIENT || - dp->estimate == ES_CALCSIZE || + + estimate = ES_CLIENT; + for (el = dp->estimatelist; el != NULL; el = el->next) { + estimate = (estimate_t)GPOINTER_TO_INT(el->data); + if (estimate == ES_SERVER) + break; + } + if (estimate == ES_SERVER) { + info_t info; + nb_server++; + get_info(dp->host->hostname, dp->name, &info); + for(i = 0; i < MAX_LEVELS; i++) { + int lev = est(dp)->estimate[i].level; + + if(lev == -1) break; + server_estimate(dp, i, &info, lev); + } + g_fprintf(stderr,_("%s time %s: got result for host %s disk %s:"), + get_pname(), walltime_str(curclock()), + dp->host->hostname, qname); + g_fprintf(stderr,_(" %d -> %lldK, %d -> %lldK, %d -> %lldK\n"), + est(dp)->estimate[0].level, + (long long)est(dp)->estimate[0].nsize, + est(dp)->estimate[1].level, + (long long)est(dp)->estimate[1].nsize, + est(dp)->estimate[2].level, + (long long)est(dp)->estimate[2].nsize); + if (!am_has_feature(hostp->features, fe_xml_estimate)) { + est(dp)->state = DISK_DONE; + remove_disk(&startq, dp); + enqueue_disk(&estq, dp); + } + } + + estimate = ES_SERVER; + for (el = dp->estimatelist; el != NULL; el = el->next) { + estimate = (estimate_t)GPOINTER_TO_INT(el->data); + if (estimate == ES_CLIENT || estimate == ES_CALCSIZE) + break; + } + if (estimate == ES_CLIENT || + estimate == ES_CALCSIZE || (am_has_feature(hostp->features, fe_req_xml) && am_has_feature(hostp->features, fe_xml_estimate))) { nb_client++; @@ -1413,23 +1575,29 @@ static void getsize( char spindle[NUM_STR_SIZE]; char *o; char *l; + info_t info; + get_info(dp->host->hostname, dp->name, &info); for(i = 0; i < MAX_LEVELS; i++) { - int lev = est(dp)->level[i]; + char *server; + int lev = est(dp)->estimate[i].level; if (lev == -1) break; g_snprintf(level, SIZEOF(level), "%d", lev); + if (am_has_feature(hostp->features, fe_xml_level_server) && + server_can_do_estimate(dp, &info, lev)) { + server = "YES"; + } else { + server = ""; + } vstrextend(&levelstr, " ", - level, + level, server, "\n", NULL); } g_snprintf(spindle, SIZEOF(spindle), "%d", dp->spindle); spindlestr = vstralloc(" ", spindle, "\n", NULL); - o = xml_optionstr(dp, hostp->features, NULL, 0); - if (o == NULL) { - error(_("problem with option string, check the dumptype definition.\n")); - } + o = xml_optionstr(dp, 0); if (strcmp(dp->program,"DUMP") == 0 || strcmp(dp->program,"GNUTAR") == 0) { @@ -1442,37 +1610,21 @@ static void getsize( " APPLICATION\n", NULL); if (dp->application) { - char *xml_app = xml_application(dp->application, - hostp->features); + application_t *application; + char *xml_app; + + application = lookup_application(dp->application); + g_assert(application != NULL); + xml_app = xml_application(dp, application, + hostp->features); vstrextend(&l, xml_app, NULL); amfree(xml_app); } } - if (am_has_feature(hostp->features, fe_xml_estimate)) { - if (dp->estimate == ES_CLIENT) { - vstrextend(&l, " CLIENT\n", - NULL); - } else if (dp->estimate == ES_SERVER) { - vstrextend(&l, " SERVER\n", - NULL); - } else if (dp->estimate == ES_CALCSIZE) { - vstrextend(&l, " CALCSIZE\n", - NULL); - } - } - if (dp->estimate == ES_CALCSIZE) { - if (!am_has_feature(hostp->features, - fe_calcsize_estimate)) { - log_add(L_WARNING, - _("%s:%s does not support CALCSIZE for estimate, using CLIENT.\n"), - hostp->hostname, qname); - dp->estimate = ES_CLIENT; - } else { - vstrextend(&l, " YES\n", - NULL); - } - } + es = xml_estimate(dp->estimatelist, hostp->features); + vstrextend(&l, es, "\n", NULL); + amfree(es); vstrextend(&l, " ", b64disk, "\n", NULL); if (dp->device) vstrextend(&l, " ", b64device, "\n", NULL); @@ -1480,6 +1632,9 @@ static void getsize( strappend(s, l); s_len += strlen(l); amfree(l); + amfree(levelstr); + amfree(spindlestr); + amfree(o); } else if (strcmp(dp->program,"DUMP") != 0 && strcmp(dp->program,"GNUTAR") != 0) { est(dp)->errstr = newvstrallocf(est(dp)->errstr, @@ -1495,7 +1650,7 @@ static void getsize( char *includefree = NULL; char spindle[NUM_STR_SIZE]; char level[NUM_STR_SIZE]; - int lev = est(dp)->level[i]; + int lev = est(dp)->estimate[i].level; if(lev == -1) break; @@ -1504,7 +1659,7 @@ static void getsize( if (am_has_feature(hostp->features, fe_sendsize_req_options)){ exclude1 = " OPTIONS |"; - exclude2 = optionstr(dp, hostp->features, NULL); + exclude2 = optionstr(dp); if ( exclude2 == NULL ) { error(_("problem with option string, check the dumptype definition.\n")); } @@ -1541,15 +1696,15 @@ static void getsize( } } - if (dp->estimate == ES_CALCSIZE && + if (estimate == ES_CALCSIZE && !am_has_feature(hostp->features, fe_calcsize_estimate)) { log_add(L_WARNING, _("%s:%s does not support CALCSIZE for estimate, using CLIENT.\n"), hostp->hostname, qname); - dp->estimate = ES_CLIENT; + estimate = ES_CLIENT; } - if(dp->estimate == ES_CLIENT) + if(estimate == ES_CLIENT) calcsize = ""; else calcsize = "CALCSIZE "; @@ -1559,7 +1714,7 @@ static void getsize( " ", qname, " ", dp->device ? qdevice : "", " ", level, - " ", est(dp)->dumpdate[i], + " ", est(dp)->estimate[i].dumpdate, " ", spindle, " ", exclude1, exclude2, ((includefree != NULL) ? " " : ""), @@ -1573,14 +1728,21 @@ static void getsize( amfree(excludefree); } } - remove_disk(&startq, dp); if (s != NULL) { estimates += i; strappend(req, s); req_len += s_len; amfree(s); - est(dp)->state = DISK_ACTIVE; - } else { + if (est(dp)->state == DISK_DONE) { + remove_disk(&estq, dp); + est(dp)->state = DISK_PARTIALY_DONE; + enqueue_disk(&pestq, dp); + } else { + remove_disk(&startq, dp); + est(dp)->state = DISK_ACTIVE; + } + } else if (est(dp)->state != DISK_DONE) { + remove_disk(&startq, dp); est(dp)->state = DISK_DONE; if (est(dp)->errstr == NULL) { est(dp)->errstr = vstrallocf( @@ -1589,112 +1751,8 @@ static void getsize( enqueue_disk(&failq, dp); } } - if (dp->estimate == ES_SERVER) { - info_t info; - nb_server++; - get_info(dp->host->hostname, dp->name, &info); - for(i = 0; i < MAX_LEVELS; i++) { - int j; - int lev = est(dp)->level[i]; - - if(lev == -1) break; - if(lev == 0) { /* use latest level 0, should do extrapolation */ - gint64 est_size = (gint64)0; - int nb_est = 0; - - for(j=NB_HISTORY-2;j>=0;j--) { - if(info.history[j].level == 0) { - if(info.history[j].size < (gint64)0) continue; - est_size = info.history[j].size; - nb_est++; - } - } - if(nb_est > 0) { - est(dp)->est_size[i] = est_size; - } - else if(info.inf[lev].size > (gint64)1000) { /* stats */ - est(dp)->est_size[i] = info.inf[lev].size; - } - else { - est(dp)->est_size[i] = (gint64)1000000; - } - } - else if(lev == est(dp)->last_level) { - /* means of all X day at the same level */ - #define NB_DAY 30 - int nb_day = 0; - gint64 est_size_day[NB_DAY]; - int nb_est_day[NB_DAY]; - for(j=0;j=0;j--) { - if(info.history[j].level <= 0) continue; - if(info.history[j].size < (gint64)0) continue; - if(info.history[j].level==info.history[j+1].level) { - if(nb_day NB_DAY-1) nb_day = NB_DAY-1; - - while(nb_day > 0 && nb_est_day[nb_day] == 0) nb_day--; - - if(nb_est_day[nb_day] > 0) { - est(dp)->est_size[i] = est_size_day[nb_day] / - (gint64)nb_est_day[nb_day]; - } - else if(info.inf[lev].size > (gint64)1000) { /* stats */ - est(dp)->est_size[i] = info.inf[lev].size; - } - else { - est(dp)->est_size[i] = (gint64)10000; - } - } - else if(lev == est(dp)->last_level + 1) { - /* means of all first day at a new level */ - gint64 est_size = (gint64)0; - int nb_est = 0; - - for(j=NB_HISTORY-2;j>=0;j--) { - if(info.history[j].level <= 0) continue; - if(info.history[j].size < (gint64)0) continue; - if(info.history[j].level == info.history[j+1].level + 1 ) { - est_size += info.history[j].size; - nb_est++; - } - } - if(nb_est > 0) { - est(dp)->est_size[i] = est_size / (gint64)nb_est; - } - else if(info.inf[lev].size > (gint64)1000) { /* stats */ - est(dp)->est_size[i] = info.inf[lev].size; - } - else { - est(dp)->est_size[i] = (gint64)100000; - } - } - } - g_fprintf(stderr,_("%s time %s: got result for host %s disk %s:"), - get_pname(), walltime_str(curclock()), - dp->host->hostname, qname); - g_fprintf(stderr,_(" %d -> %lldK, %d -> %lldK, %d -> %lldK\n"), - est(dp)->level[0], (long long)est(dp)->est_size[0], - est(dp)->level[1], (long long)est(dp)->est_size[1], - est(dp)->level[2], (long long)est(dp)->est_size[2]); - if (!am_has_feature(hostp->features, fe_xml_estimate)) { - est(dp)->state = DISK_DONE; - remove_disk(&startq, dp); - enqueue_disk(&estq, dp); - } - } + amfree(b64disk); + amfree(b64device); amfree(qname); amfree(qdevice); } @@ -1723,12 +1781,13 @@ static void getsize( timeout = (time_t)getconf_int(CNF_CTIMEOUT); } - secdrv = security_getdriver(hostp->disks->security_driver); + dbprintf(_("send request:\n----\n%s\n----\n\n"), req); + secdrv = security_getdriver(hostp->disks->auth); if (secdrv == NULL) { hostp->up = HOST_DONE; log_add(L_ERROR, _("Could not find security driver '%s' for host '%s'"), - hostp->disks->security_driver, hostp->hostname); + hostp->disks->auth, hostp->hostname); amfree(req); return; } @@ -1788,8 +1847,12 @@ static void handle_result( hostp->up = HOST_READY; if (pkt == NULL) { - errbuf = vstrallocf(_("Request to %s failed: %s"), + if (strcmp(security_geterror(sech), "timeout waiting for REP") == 0) { + errbuf = vstrallocf("Some estimate timeout on %s, using server estimate if possible", hostp->hostname); + } else { + errbuf = vstrallocf(_("Request to %s failed: %s"), hostp->hostname, security_geterror(sech)); + } goto error_return; } if (pkt->type == P_NAK) { @@ -1814,6 +1877,7 @@ static void handle_result( } } + dbprintf(_("got reply:\n----\n%s\n----\n\n"), pkt->body); s = pkt->body; ch = *s++; while(ch) { @@ -1822,6 +1886,9 @@ static void handle_result( if(strncmp_const(line, "OPTIONS ") == 0) { t = strstr(line, "features="); if(t != NULL && (g_ascii_isspace((int)t[-1]) || t[-1] == ';')) { + char *u = strchr(t, ';'); + if (u) + *u = '\0'; t += SIZEOF("features=")-1; am_release_feature_set(hostp->features); if((hostp->features = am_string_to_feature(t)) == NULL) { @@ -1829,6 +1896,8 @@ static void handle_result( _(": bad features value: %s\n"), line); goto error_return; } + if (u) + *u = ';'; } skip_quoted_line(s, ch); continue; @@ -1855,7 +1924,7 @@ static void handle_result( skip_quoted_line(s, ch); continue; } - t = index(t,'\n'); + t = strchr(t,'\n'); if (t) /* truncate after the first line */ *t = '\0'; errbuf = vstralloc(hostp->hostname, @@ -1881,7 +1950,6 @@ static void handle_result( skip_integer(t, tch); skip_whitespace(t, tch); - dp = lookup_hostdisk(hostp, disk); dp = lookup_hostdisk(hostp, disk); if(dp == NULL) { log_add(L_ERROR, _("%s: invalid reply from sendsize: `%s'\n"), @@ -1913,32 +1981,25 @@ static void handle_result( amfree(disk); - if (dp->estimate == ES_SERVER) { - if (size == (gint64)-2) { - for(i = 0; i < MAX_LEVELS; i++) { - if (est(dp)->level[i] == level) { - est(dp)->est_size[i] = -1; /* remove estimate */ - break; - } - } - if(i == MAX_LEVELS) { - goto bad_msg; /* this est wasn't requested */ + for (i = 0; i < MAX_LEVELS; i++) { + if (est(dp)->estimate[i].level == level) { + if (size == (gint64)-2) { + est(dp)->estimate[i].nsize = -1; /* remove estimate */ + est(dp)->estimate[i].guessed = 0; + } else if (size > (gint64)-1) { + /* take the size returned by the client */ + est(dp)->estimate[i].nsize = size; + est(dp)->estimate[i].guessed = 0; } - - } - est(dp)->got_estimate++; - } else if (size > (gint64)-1) { - for(i = 0; i < MAX_LEVELS; i++) { - if(est(dp)->level[i] == level) { - est(dp)->est_size[i] = size; - break; - } - } - if(i == MAX_LEVELS) { - goto bad_msg; /* this est wasn't requested */ + break; } - est(dp)->got_estimate++; } + if (i == MAX_LEVELS && level > 0) { + /* client always report level 0 for some error */ + goto bad_msg; /* this est wasn't requested */ + } + est(dp)->got_estimate++; + s = t; ch = tch; skip_quoted_line(s, ch); @@ -1977,7 +2038,7 @@ static void handle_result( est(dp)->state = DISK_PARTIALY_DONE; } - if(est(dp)->level[0] == -1) continue; /* ignore this disk */ + if (est(dp)->estimate[0].level == -1) continue; /* ignore this disk */ qname = quote_string(dp->name); @@ -1986,9 +2047,12 @@ static void handle_result( get_pname(), walltime_str(curclock()), dp->host->hostname, qname); g_fprintf(stderr,_(" %d -> %lldK, %d -> %lldK, %d -> %lldK\n"), - est(dp)->level[0], (long long)est(dp)->est_size[0], - est(dp)->level[1], (long long)est(dp)->est_size[1], - est(dp)->level[2], (long long)est(dp)->est_size[2]); + est(dp)->estimate[0].level, + (long long)est(dp)->estimate[0].nsize, + est(dp)->estimate[1].level, + (long long)est(dp)->estimate[1].nsize, + est(dp)->estimate[2].level, + (long long)est(dp)->estimate[2].nsize); enqueue_disk(&pestq, dp); } else if(pkt->type == P_REP) { @@ -1996,37 +2060,30 @@ static void handle_result( get_pname(), walltime_str(curclock()), dp->host->hostname, qname); g_fprintf(stderr,_(" %d -> %lldK, %d -> %lldK, %d -> %lldK\n"), - est(dp)->level[0], (long long)est(dp)->est_size[0], - est(dp)->level[1], (long long)est(dp)->est_size[1], - est(dp)->level[2], (long long)est(dp)->est_size[2]); - if((est(dp)->level[0] != -1 && est(dp)->est_size[0] > (gint64)0) || - (est(dp)->level[1] != -1 && est(dp)->est_size[1] > (gint64)0) || - (est(dp)->level[2] != -1 && est(dp)->est_size[2] > (gint64)0)) { - - if(est(dp)->level[2] != -1 && est(dp)->est_size[2] < (gint64)0) { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, est(dp)->level[2]); - est(dp)->level[2] = -1; - } - if(est(dp)->level[1] != -1 && est(dp)->est_size[1] < (gint64)0) { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, - est(dp)->level[1]); - est(dp)->level[1] = -1; - } - if(est(dp)->level[0] != -1 && est(dp)->est_size[0] < (gint64)0) { - log_add(L_WARNING, - _("disk %s:%s, estimate of level %d failed."), - dp->host->hostname, qname, est(dp)->level[0]); - est(dp)->level[0] = -1; + est(dp)->estimate[0].level, + (long long)est(dp)->estimate[0].nsize, + est(dp)->estimate[1].level, + (long long)est(dp)->estimate[1].nsize, + est(dp)->estimate[2].level, + (long long)est(dp)->estimate[2].nsize); + if ((est(dp)->estimate[0].level != -1 && + est(dp)->estimate[0].nsize > (gint64)0) || + (est(dp)->estimate[1].level != -1 && + est(dp)->estimate[1].nsize > (gint64)0) || + (est(dp)->estimate[2].level != -1 && + est(dp)->estimate[2].nsize > (gint64)0)) { + + for (i=MAX_LEVELS-1; i >=0; i--) { + if (est(dp)->estimate[i].level != -1 && + est(dp)->estimate[i].nsize < (gint64)0) { + est(dp)->estimate[i].level = -1; + } } enqueue_disk(&estq, dp); } else { enqueue_disk(&failq, dp); - if(est(dp)->got_estimate) { + if(est(dp)->got_estimate && !est(dp)->errstr) { est(dp)->errstr = vstrallocf("disk %s, all estimate failed", qname); } @@ -2044,23 +2101,24 @@ static void handle_result( } if (est(dp)->post_dle == 0 && (pkt->type == P_REP || - ((est(dp)->level[0] == -1 || est(dp)->est_size[0] > (gint64)0) && - (est(dp)->level[1] == -1 || est(dp)->est_size[1] > (gint64)0) && - (est(dp)->level[2] == -1 || est(dp)->est_size[2] > (gint64)0)))) { - run_server_scripts(EXECUTE_ON_POST_DLE_ESTIMATE, - get_config_name(), dp, est(dp)->level[0]); + ((est(dp)->estimate[0].level == -1 || + est(dp)->estimate[0].nsize > (gint64)0) && + (est(dp)->estimate[1].level == -1 || + est(dp)->estimate[1].nsize > (gint64)0) && + (est(dp)->estimate[2].level == -1 || + est(dp)->estimate[2].nsize > (gint64)0)))) { + run_server_dle_scripts(EXECUTE_ON_POST_DLE_ESTIMATE, + get_config_name(), dp, + est(dp)->estimate[0].level); est(dp)->post_dle = 1; } amfree(qname); } if(hostp->up == HOST_DONE) { - for(dp = hostp->disks; dp != NULL; dp = dp->hostnext) { - if (dp->todo) - if (pkt->type == P_REP) { - run_server_scripts(EXECUTE_ON_POST_HOST_ESTIMATE, - get_config_name(), dp, est(dp)->level[0]); - } + if (pkt->type == P_REP) { + run_server_host_scripts(EXECUTE_ON_POST_HOST_ESTIMATE, + get_config_name(), hostp); } } @@ -2108,6 +2166,19 @@ static void handle_result( * reported. */ log_add(L_ERROR, "%s", errbuf); + for(dp = hostp->disks; dp != NULL; dp = dp->hostnext) { + if (dp->todo) { + qname = quote_string(dp->name); + est(dp)->state = DISK_DONE; + remove_disk(&waitq, dp); + enqueue_disk(&failq, dp); + + est(dp)->errstr = g_strdup(errbuf); + g_fprintf(stderr, _("error result for host %s disk %s: %s\n"), + dp->host->hostname, qname, errbuf); + amfree(qname); + } + } } hostp->up = HOST_DONE; amfree(errbuf); @@ -2126,7 +2197,7 @@ static void handle_result( */ static int schedule_order(disk_t *a, disk_t *b); /* subroutines */ -static int pick_inclevel(disk_t *dp); +static one_est_t *pick_inclevel(disk_t *dp); static void analyze_estimate( disk_t *dp) @@ -2147,59 +2218,46 @@ static void analyze_estimate( have_info = 1; } - ep->degr_level = -1; - ep->degr_nsize = (gint64)-1; - ep->degr_csize = (gint64)-1; + ep->degr_est = &default_one_est; - if(ep->next_level0 <= 0 || (have_info && ep->last_level == 0 - && (info.command & FORCE_NO_BUMP))) { - if(ep->next_level0 <= 0) { + if (ep->next_level0 <= 0 || (have_info && ep->last_level == 0 + && (ISSET(info.command, FORCE_NO_BUMP)))) { + if (ep->next_level0 <= 0) { g_fprintf(stderr,_("(due for level 0) ")); } - ep->dump_level = 0; - ep->dump_nsize = est_size(dp, 0); - ep->dump_csize = est_tape_size(dp, 0); - if(ep->dump_csize <= (gint64)0) { + ep->dump_est = est_for_level(dp, 0); + if (ep->dump_est->csize <= (gint64)0) { g_fprintf(stderr, _("(no estimate for level 0, picking an incr level)\n")); - ep->dump_level = pick_inclevel(dp); - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); - - if(ep->dump_nsize == (gint64)-1) { - ep->dump_level = ep->dump_level + 1; - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); + ep->dump_est = pick_inclevel(dp); + + if (ep->dump_est->nsize == (gint64)-1) { + ep->dump_est = est_for_level(dp, ep->dump_est->level + 1); } } else { - total_lev0 += (double) ep->dump_csize; + total_lev0 += (double) ep->dump_est->csize; if(ep->last_level == -1 || dp->skip_incr) { g_fprintf(stderr,_("(%s disk, can't switch to degraded mode)\n"), dp->skip_incr? "skip-incr":_("new")); if (dp->skip_incr && ep->degr_mesg == NULL) { - ep->degr_mesg = _("Can't switch to degraded mode when using a skip-incr disk"); + ep->degr_mesg = _("Skpping: skip-incr disk can't be dumped in degraded mode"); } - ep->degr_level = -1; - ep->degr_nsize = (gint64)-1; - ep->degr_csize = (gint64)-1; + ep->degr_est = &default_one_est; } else { /* fill in degraded mode info */ g_fprintf(stderr,_("(picking inclevel for degraded mode)")); - ep->degr_level = pick_inclevel(dp); - ep->degr_nsize = est_size(dp, ep->degr_level); - ep->degr_csize = est_tape_size(dp, ep->degr_level); - if(ep->degr_csize == (gint64)-1) { - ep->degr_level = ep->degr_level + 1; - ep->degr_nsize = est_size(dp, ep->degr_level); - ep->degr_csize = est_tape_size(dp, ep->degr_level); + ep->degr_est = pick_inclevel(dp); + if (ep->degr_est->level >= 0 && + ep->degr_est->csize == (gint64)-1) { + ep->degr_est = est_for_level(dp, ep->degr_est->level + 1); } - if(ep->degr_csize == (gint64)-1) { + if (ep->degr_est->csize == (gint64)-1) { g_fprintf(stderr,_("(no inc estimate)")); if (ep->degr_mesg == NULL) - ep->degr_mesg = _("Can't switch to degraded mode because an incremental estimate could not be performed"); - ep->degr_level = -1; + ep->degr_mesg = _("Skipping: an incremental estimate could not be performed, so disk cannot be dumped in degraded mode"); + ep->degr_est = &default_one_est; } g_fprintf(stderr,"\n"); } @@ -2208,37 +2266,47 @@ static void analyze_estimate( else { g_fprintf(stderr,_("(not due for a full dump, picking an incr level)\n")); /* XXX - if this returns -1 may be we should force a total? */ - ep->dump_level = pick_inclevel(dp); - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); + ep->dump_est = pick_inclevel(dp); - if(ep->dump_csize == (gint64)-1) { - ep->dump_level = ep->last_level; - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); + if (ep->dump_est->csize == (gint64)-1) { + ep->dump_est = est_for_level(dp, ep->last_level); } - if(ep->dump_csize == (gint64)-1) { - ep->dump_level = ep->last_level + 1; - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); + if (ep->dump_est->csize == (gint64)-1) { + ep->dump_est = est_for_level(dp, ep->last_level + 1); } - if(ep->dump_csize == (gint64)-1) { - ep->dump_level = 0; - ep->dump_nsize = est_size(dp, ep->dump_level); - ep->dump_csize = est_tape_size(dp, ep->dump_level); + if (ep->dump_est->csize == (gint64)-1) { + ep->dump_est = est_for_level(dp, 0); } if (ep->degr_mesg == NULL) { - ep->degr_mesg = _("Can't switch to degraded mode because a full is not planned"); + ep->degr_mesg = _("Skipping: a full is not planned, so can't dump in degraded mode"); + } + } + + if (ep->dump_est->level < 0) { + int i; + char *q = quote_string("no estimate"); + + g_fprintf(stderr,_(" no valid estimate\n")); + for(i=0; iestimate[i].level >= 0) { + g_fprintf(stderr,(" level: %d nsize: %lld csize: %lld\n"), + est(dp)->estimate[i].level, + (long long)est(dp)->estimate[i].nsize, + (long long)est(dp)->estimate[i].csize); + } } + log_add(L_WARNING, _("%s %s %s 0 %s"), dp->host->hostname, qname, + planner_timestamp, q); + amfree(q); } g_fprintf(stderr,_(" curr level %d nsize %lld csize %lld "), - ep->dump_level, (long long)ep->dump_nsize, - (long long)ep->dump_csize); + ep->dump_est->level, (long long)ep->dump_est->nsize, + (long long)ep->dump_est->csize); insert_disk(&schedq, dp, schedule_order); - total_size += (gint64)tt_blocksize_kb + ep->dump_csize + tape_mark; + total_size += (gint64)tt_blocksize_kb + ep->dump_est->csize + tape_mark; /* update the balanced size */ if(!(dp->skip_full || dp->strategy == DS_NOFULL || @@ -2248,7 +2316,15 @@ static void analyze_estimate( lev0size = est_tape_size(dp, 0); if(lev0size == (gint64)-1) lev0size = ep->last_lev0size; - balanced_size += (double)(lev0size / (gint64)runs_per_cycle); + if (dp->strategy == DS_NOINC) { + balanced_size += (double)lev0size; + } else if (dp->dumpcycle == 0) { + balanced_size += (double)(lev0size * conf_dumpcycle / (gint64)runs_per_cycle); + } else if (dp->dumpcycle != conf_dumpcycle) { + balanced_size += (double)(lev0size * (conf_dumpcycle / dp->dumpcycle) / (gint64)runs_per_cycle); + } else { + balanced_size += (double)(lev0size / (gint64)runs_per_cycle); + } } g_fprintf(stderr,_("total size %lld total_lev0 %1.0lf balanced-lev0size %1.0lf\n"), @@ -2258,8 +2334,16 @@ static void analyze_estimate( /* It can be an error from a script */ if (est(dp)->errstr) { char *qerrstr = quote_string(est(dp)->errstr); - log_add(L_FAIL, _("%s %s %s 0 %s"), dp->host->hostname, qname, - planner_timestamp, qerrstr); + /* Log only a warning if a server estimate is available */ + if (est(dp)->estimate[0].nsize > 0 || + est(dp)->estimate[1].nsize > 0 || + est(dp)->estimate[2].nsize > 0) { + log_add(L_WARNING, _("%s %s %s 0 %s"), dp->host->hostname, qname, + planner_timestamp, qerrstr); + } else { + log_add(L_FAIL, _("%s %s %s 0 %s"), dp->host->hostname, qname, + planner_timestamp, qerrstr); + } amfree(qerrstr); } @@ -2304,78 +2388,79 @@ static int schedule_order( diff = est(b)->dump_priority - est(a)->dump_priority; if(diff != 0) return diff; - ldiff = est(b)->dump_csize - est(a)->dump_csize; + ldiff = est(b)->dump_est->csize - est(a)->dump_est->csize; if(ldiff < (gint64)0) return -1; /* XXX - there has to be a better way to dothis */ if(ldiff > (gint64)0) return 1; return 0; } -static int pick_inclevel( +static one_est_t *pick_inclevel( disk_t *dp) { - int base_level, bump_level; - gint64 base_size, bump_size; + one_est_t *level0_est, *base_est, *bump_est; gint64 thresh; char *qname; - base_level = est(dp)->last_level; + level0_est = est_for_level(dp, 0); + base_est = est_for_level(dp, est(dp)->last_level); /* if last night was level 0, do level 1 tonight, no ifs or buts */ - if(base_level == 0) { + if (base_est->level == 0) { g_fprintf(stderr,_(" picklev: last night 0, so tonight level 1\n")); - return 1; + return est_for_level(dp, 1); } /* if no-full option set, always do level 1 */ if(dp->strategy == DS_NOFULL) { g_fprintf(stderr,_(" picklev: no-full set, so always level 1\n")); - return 1; + return est_for_level(dp, 1); } - base_size = est_size(dp, base_level); - /* if we didn't get an estimate, we can't do an inc */ - if(base_size == (gint64)-1) { - base_size = est_size(dp, base_level+1); - if(base_size > (gint64)0) /* FORCE_BUMP */ - return base_level+1; - g_fprintf(stderr,_(" picklev: no estimate for level %d, so no incs\n"), base_level); - return base_level; + if (base_est->nsize == (gint64)-1) { + bump_est = est_for_level(dp, est(dp)->last_level + 1); + if (bump_est->nsize > (gint64)0) { /* FORCE_BUMP */ + g_fprintf(stderr,_(" picklev: bumping to level %d\n"), bump_est->level); + return bump_est; + } + g_fprintf(stderr,_(" picklev: no estimate for level %d, so no incs\n"), base_est->level); + return base_est; } - thresh = bump_thresh(base_level, est_size(dp, 0), dp->bumppercent, dp->bumpsize, dp->bumpmult); + thresh = bump_thresh(base_est->level, level0_est->nsize, dp->bumppercent, + dp->bumpsize, dp->bumpmult); g_fprintf(stderr, _(" pick: size %lld level %d days %d (thresh %lldK, %d days)\n"), - (long long)base_size, base_level, est(dp)->level_days, + (long long)base_est->nsize, base_est->level, est(dp)->level_days, (long long)thresh, dp->bumpdays); - if(base_level == 9 + if(base_est->level == (DUMP_LEVELS - 1) || est(dp)->level_days < dp->bumpdays - || base_size <= thresh) - return base_level; + || base_est->nsize <= thresh) + return base_est; - bump_level = base_level + 1; - bump_size = est_size(dp, bump_level); + bump_est = est_for_level(dp, base_est->level + 1); - if(bump_size == (gint64)-1) return base_level; + if (bump_est->nsize == (gint64)-1) + return base_est; g_fprintf(stderr, _(" pick: next size %lld... "), - (long long)bump_size); + (long long)bump_est->nsize); - if(base_size - bump_size < thresh) { + if (base_est->nsize - bump_est->nsize < thresh) { g_fprintf(stderr, _("not bumped\n")); - return base_level; + return base_est; } qname = quote_string(dp->name); g_fprintf(stderr, _("BUMPED\n")); log_add(L_INFO, _("Incremental of %s:%s bumped to level %d."), - dp->host->hostname, qname, bump_level); + dp->host->hostname, qname, bump_est->level); amfree(qname); - return bump_level; + return bump_est; } @@ -2415,6 +2500,7 @@ static void delay_dumps(void) disk_t * dp; disk_t * ndp; disk_t * preserve; + disk_t * delayed_dp; bi_t * bi; bi_t * nbi; gint64 new_total; /* New total_size */ @@ -2424,6 +2510,8 @@ static void delay_dumps(void) int delete; char * message; gint64 full_size; + time_t timestamps; + int priority; biq.head = biq.tail = NULL; @@ -2440,7 +2528,7 @@ static void delay_dumps(void) for(dp = schedq.head; dp != NULL; dp = ndp) { int avail_tapes = 1; - if (dp->tape_splitsize > (gint64)0) + if (dp->splitsize > (gint64)0 || dp->allow_split) avail_tapes = conf_runtapes; ndp = dp->next; /* remove_disk zaps this */ @@ -2448,7 +2536,7 @@ static void delay_dumps(void) full_size = est_tape_size(dp, 0); if (full_size > tapetype_get_length(tape) * (gint64)avail_tapes) { char *qname = quote_string(dp->name); - if (conf_runtapes > 1 && dp->tape_splitsize == (gint64)0) { + if (conf_runtapes > 1 && dp->splitsize == (gint64)0) { log_add(L_WARNING, _("disk %s:%s, full dump (%lldKB) will be larger than available tape space" ", you could define a splitsize"), dp->host->hostname, qname, @@ -2461,16 +2549,16 @@ static void delay_dumps(void) amfree(qname); } - if (est(dp)->dump_csize == (gint64)-1 || - est(dp)->dump_csize <= tapetype_get_length(tape) * (gint64)avail_tapes) { + if (est(dp)->dump_est->csize == (gint64)-1 || + est(dp)->dump_est->csize <= tapetype_get_length(tape) * (gint64)avail_tapes) { continue; } /* Format dumpsize for messages */ g_snprintf(est_kb, 20, "%lld KB,", - (long long)est(dp)->dump_csize); + (long long)est(dp)->dump_est->csize); - if(est(dp)->dump_level == 0) { + if(est(dp)->dump_est->level == 0) { if(dp->skip_incr) { delete = 1; message = _("but cannot incremental dump skip-incr disk"); @@ -2479,17 +2567,17 @@ static void delay_dumps(void) delete = 1; message = _("but cannot incremental dump new disk"); } - else if(est(dp)->degr_level < 0) { + else if(est(dp)->degr_est->level < 0) { delete = 1; message = _("but no incremental estimate"); } - else if (est(dp)->degr_csize > tapetype_get_length(tape)) { + else if (est(dp)->degr_est->csize > tapetype_get_length(tape)) { delete = 1; message = _("incremental dump also larger than tape"); } else { delete = 0; - message = _("full dump delayed"); + message = _("full dump delayed, doing incremental"); } } else { @@ -2515,51 +2603,71 @@ static void delay_dumps(void) nb_forced_level_0 = 0; preserve = NULL; - for(dp = schedq.head; dp != NULL && preserve == NULL; dp = dp->next) - if(est(dp)->dump_level == 0) - preserve = dp; + timestamps = 2147483647; + priority = 0; + for(dp = schedq.head; dp != NULL; dp = dp->next) { + if (est(dp)->dump_est->level == 0) { + if (!preserve || + est(dp)->dump_priority > priority || + (est(dp)->dump_priority == priority && + est(dp)->info->inf[0].date < timestamps)) { + priority = est(dp)->dump_priority; + timestamps = est(dp)->info->inf[0].date; + preserve = dp; + } + } + } /* 2.a. Do not delay forced full */ - for(dp = schedq.tail; + delayed_dp = NULL; + do { + delayed_dp = 0; + timestamps = 0; + for(dp = schedq.tail; dp != NULL && total_size > tape_length; dp = ndp) { - ndp = dp->prev; + ndp = dp->prev; - if(est(dp)->dump_level != 0) continue; + if(est(dp)->dump_est->level != 0) continue; - get_info(dp->host->hostname, dp->name, &info); - if(info.command & FORCE_FULL) { - nb_forced_level_0 += 1; - preserve = dp; - continue; - } - - if(dp != preserve) { + get_info(dp->host->hostname, dp->name, &info); + if(ISSET(info.command, FORCE_FULL)) { + nb_forced_level_0 += 1; + preserve = dp; + continue; + } + if (dp != preserve && + est(dp)->info->inf[0].date > timestamps) { + delayed_dp = dp; + timestamps = est(dp)->info->inf[0].date; + } + } + if (delayed_dp) { /* Format dumpsize for messages */ g_snprintf(est_kb, 20, "%lld KB,", - (long long)est(dp)->dump_csize); + (long long)est(delayed_dp)->dump_est->csize); - if(dp->skip_incr) { + if(delayed_dp->skip_incr) { delete = 1; message = _("but cannot incremental dump skip-incr disk"); } - else if(est(dp)->last_level < 0) { + else if(est(delayed_dp)->last_level < 0) { delete = 1; message = _("but cannot incremental dump new disk"); } - else if(est(dp)->degr_level < 0) { + else if(est(delayed_dp)->degr_est->level < 0) { delete = 1; message = _("but no incremental estimate"); } else { delete = 0; - message = _("full dump delayed"); + message = _("full dump delayed, doing incremental"); } - delay_one_dump(dp, delete, _("dumps too big,"), est_kb, + delay_one_dump(delayed_dp, delete, _("dumps too big,"), est_kb, message, NULL); } - } + } while (delayed_dp); /* 2.b. Delay forced full if needed */ if(nb_forced_level_0 > 0 && total_size > tape_length) { @@ -2568,11 +2676,11 @@ static void delay_dumps(void) dp = ndp) { ndp = dp->prev; - if(est(dp)->dump_level == 0 && dp != preserve) { + if(est(dp)->dump_est->level == 0 && dp != preserve) { /* Format dumpsize for messages */ g_snprintf(est_kb, 20, "%lld KB,", - (long long)est(dp)->dump_csize); + (long long)est(dp)->dump_est->csize); if(dp->skip_incr) { delete = 1; @@ -2582,7 +2690,7 @@ static void delay_dumps(void) delete = 1; message = _("but cannot incremental dump new disk"); } - else if(est(dp)->degr_level < 0) { + else if(est(dp)->degr_est->level < 0) { delete = 1; message = _("but no incremental estimate"); } @@ -2609,11 +2717,11 @@ static void delay_dumps(void) dp = ndp) { ndp = dp->prev; - if(est(dp)->dump_level != 0) { + if(est(dp)->dump_est->level != 0) { /* Format dumpsize for messages */ g_snprintf(est_kb, 20, "%lld KB,", - (long long)est(dp)->dump_csize); + (long long)est(dp)->dump_est->csize); delay_one_dump(dp, 1, _("dumps way too big,"), @@ -2636,14 +2744,14 @@ static void delay_dumps(void) int avail_tapes = 1; nbi = bi->prev; dp = bi->dp; - if(dp->tape_splitsize > (gint64)0) + if(dp->splitsize > (gint64)0) avail_tapes = conf_runtapes; if(bi->deleted) { new_total = total_size + (gint64)tt_blocksize_kb + bi->csize + (gint64)tape_mark; } else { - new_total = total_size - est(dp)->dump_csize + bi->csize; + new_total = total_size - est(dp)->dump_est->csize + bi->csize; } if((new_total <= tape_length) && (bi->csize < (tapetype_get_length(tape) * (gint64)avail_tapes))) { @@ -2656,9 +2764,7 @@ static void delay_dumps(void) insert_disk(&schedq, dp, schedule_order); } else { - est(dp)->dump_level = bi->level; - est(dp)->dump_nsize = bi->nsize; - est(dp)->dump_csize = bi->csize; + est(dp)->dump_est = est_for_level(dp, bi->level); } /* Keep it clean */ @@ -2693,7 +2799,7 @@ static void delay_dumps(void) else { dp = bi->dp; g_fprintf(stderr, _(" delay: %s now at level %d\n"), - bi->errstr, est(dp)->dump_level); + bi->errstr, est(dp)->dump_est->level); log_add(L_INFO, "%s", bi->errstr); } /*@ignore@*/ @@ -2728,9 +2834,9 @@ arglist_function1( arglist_start(argp, delete); - total_size -= (gint64)tt_blocksize_kb + est(dp)->dump_csize + (gint64)tape_mark; - if(est(dp)->dump_level == 0) { - total_lev0 -= (double) est(dp)->dump_csize; + total_size -= (gint64)tt_blocksize_kb + est(dp)->dump_est->csize + (gint64)tape_mark; + if(est(dp)->dump_est->level == 0) { + total_lev0 -= (double) est(dp)->dump_est->csize; } bi = alloc(SIZEOF(bi_t)); @@ -2744,11 +2850,11 @@ arglist_function1( bi->deleted = delete; bi->dp = dp; - bi->level = est(dp)->dump_level; - bi->nsize = est(dp)->dump_nsize; - bi->csize = est(dp)->dump_csize; + bi->level = est(dp)->dump_est->level; + bi->nsize = est(dp)->dump_est->nsize; + bi->csize = est(dp)->dump_est->csize; - g_snprintf(level_str, SIZEOF(level_str), "%d", est(dp)->dump_level); + g_snprintf(level_str, SIZEOF(level_str), "%d", est(dp)->dump_est->level); bi->errstr = vstralloc(dp->host->hostname, " ", qname, " ", planner_timestamp ? planner_timestamp : "?", @@ -2770,10 +2876,8 @@ arglist_function1( if (delete) { remove_disk(&schedq, dp); } else { - est(dp)->dump_level = est(dp)->degr_level; - est(dp)->dump_nsize = est(dp)->degr_nsize; - est(dp)->dump_csize = est(dp)->degr_csize; - total_size += (gint64)tt_blocksize_kb + est(dp)->dump_csize + (gint64)tape_mark; + est(dp)->dump_est = est(dp)->degr_est; + total_size += (gint64)tt_blocksize_kb + est(dp)->dump_est->csize + (gint64)tape_mark; } amfree(qname); return; @@ -2783,7 +2887,7 @@ arglist_function1( static int promote_highest_priority_incremental(void) { disk_t *dp, *dp1, *dp_promote; - gint64 new_size, new_total, new_lev0; + gint64 new_total, new_lev0; int check_days; int nb_today, nb_same_day, nb_today2; int nb_disk_today, nb_disk_same_day; @@ -2796,10 +2900,10 @@ static int promote_highest_priority_incremental(void) dp_promote = NULL; for(dp = schedq.head; dp != NULL; dp = dp->next) { - + one_est_t *level0_est = est_for_level(dp, 0); est(dp)->promote = -1000; - if(est_size(dp,0) <= (gint64)0) + if (level0_est->nsize <= (gint64)0) continue; if(est(dp)->next_level0 <= 0) @@ -2808,21 +2912,20 @@ static int promote_highest_priority_incremental(void) if(est(dp)->next_level0 > dp->maxpromoteday) continue; - new_size = est_tape_size(dp, 0); - new_total = total_size - est(dp)->dump_csize + new_size; - new_lev0 = (gint64)total_lev0 + new_size; + new_total = total_size - est(dp)->dump_est->csize + level0_est->csize; + new_lev0 = (gint64)total_lev0 + level0_est->csize; nb_today = 0; nb_same_day = 0; nb_disk_today = 0; nb_disk_same_day = 0; for(dp1 = schedq.head; dp1 != NULL; dp1 = dp1->next) { - if(est(dp1)->dump_level == 0) + if(est(dp1)->dump_est->level == 0) nb_disk_today++; else if(est(dp1)->next_level0 == est(dp)->next_level0) nb_disk_same_day++; if(strcmp(dp->host->hostname, dp1->host->hostname) == 0) { - if(est(dp1)->dump_level == 0) + if(est(dp1)->dump_est->level == 0) nb_today++; else if(est(dp1)->next_level0 == est(dp)->next_level0) nb_same_day++; @@ -2870,22 +2973,19 @@ static int promote_highest_priority_incremental(void) } if(dp_promote) { + one_est_t *level0_est; dp = dp_promote; + level0_est = est_for_level(dp, 0); qname = quote_string(dp->name); - new_size = est_tape_size(dp, 0); - new_total = total_size - est(dp)->dump_csize + new_size; - new_lev0 = (gint64)total_lev0 + new_size; + new_total = total_size - est(dp)->dump_est->csize + level0_est->csize; + new_lev0 = (gint64)total_lev0 + level0_est->csize; total_size = new_total; total_lev0 = (double)new_lev0; check_days = est(dp)->next_level0; - est(dp)->degr_level = est(dp)->dump_level; - est(dp)->degr_nsize = est(dp)->dump_nsize; - est(dp)->degr_csize = est(dp)->dump_csize; - est(dp)->dump_level = 0; - est(dp)->dump_nsize = est_size(dp, 0); - est(dp)->dump_csize = new_size; + est(dp)->degr_est = est(dp)->dump_est; + est(dp)->dump_est = level0_est; est(dp)->next_level0 = 0; g_fprintf(stderr, @@ -2915,7 +3015,6 @@ static int promote_hills(void) int days; int hill_days = 0; gint64 hill_size; - gint64 new_size; gint64 new_total; int my_dumpcycle; char *qname; @@ -2937,7 +3036,8 @@ static int promote_hills(void) } for(dp = schedq.head; dp != NULL; dp = dp->next) { - days = est(dp)->next_level0; /* This is > 0 by definition */ + days = est(dp)->next_level0; + if (days < 0) days = 0; if(daysskip_full && dp->strategy != DS_NOFULL && dp->strategy != DS_INCRONLY) { sp[days].disks++; @@ -2960,27 +3060,26 @@ static int promote_hills(void) /* Find all the dumps in that hill and try and remove one */ for(dp = schedq.head; dp != NULL; dp = dp->next) { + one_est_t *level0_est; if(est(dp)->next_level0 != hill_days || est(dp)->next_level0 > dp->maxpromoteday || dp->skip_full || dp->strategy == DS_NOFULL || dp->strategy == DS_INCRONLY) continue; - new_size = est_tape_size(dp, 0); - new_total = total_size - est(dp)->dump_csize + new_size; + level0_est = est_for_level(dp, 0); + if (level0_est->nsize <= (gint64)0) + continue; + new_total = total_size - est(dp)->dump_est->csize + level0_est->csize; if(new_total > tape_length) continue; /* We found a disk we can promote */ qname = quote_string(dp->name); total_size = new_total; - total_lev0 += (double)new_size; - est(dp)->degr_level = est(dp)->dump_level; - est(dp)->degr_nsize = est(dp)->dump_nsize; - est(dp)->degr_csize = est(dp)->dump_csize; - est(dp)->dump_level = 0; + total_lev0 += (double)level0_est->csize; + est(dp)->degr_est = est(dp)->dump_est; + est(dp)->dump_est = level0_est; est(dp)->next_level0 = 0; - est(dp)->dump_nsize = est_size(dp, 0); - est(dp)->dump_csize = new_size; g_fprintf(stderr, _(" promote: moving %s:%s up, total_lev0 %1.0lf, total_size %lld\n"), @@ -3031,54 +3130,48 @@ static void output_scheduleline( char degr_kps_str[NUM_STR_SIZE]; char *dump_date, *degr_date; char *features; - int i; char *qname = quote_string(dp->name); ep = est(dp); - if(ep->dump_csize == (gint64)-1) { + if(ep->dump_est->csize == (gint64)-1) { /* no estimate, fail the disk */ g_fprintf(stderr, _("%s: FAILED %s %s %s %d \"[no estimate]\"\n"), get_pname(), - dp->host->hostname, qname, planner_timestamp, ep->dump_level); + dp->host->hostname, qname, planner_timestamp, ep->dump_est->level); log_add(L_FAIL, _("%s %s %s %d [no estimate]"), - dp->host->hostname, qname, planner_timestamp, ep->dump_level); + dp->host->hostname, qname, planner_timestamp, ep->dump_est->level); amfree(qname); return; } - dump_date = degr_date = (char *)0; - for(i = 0; i < MAX_LEVELS; i++) { - if(ep->dump_level == ep->level[i]) - dump_date = ep->dumpdate[i]; - if(ep->degr_level == ep->level[i]) - degr_date = ep->dumpdate[i]; - } + dump_date = ep->dump_est->dumpdate; + degr_date = ep->degr_est->dumpdate; #define fix_rate(rate) (rate < 1.0 ? DEFAULT_DUMPRATE : rate) - if(ep->dump_level == 0) { + if(ep->dump_est->level == 0) { dump_kps = fix_rate(ep->fullrate); - dump_time = (time_t)((double)ep->dump_csize / dump_kps); + dump_time = (time_t)((double)ep->dump_est->csize / dump_kps); - if(ep->degr_csize != (gint64)-1) { + if(ep->degr_est->csize != (gint64)-1) { degr_kps = fix_rate(ep->incrrate); - degr_time = (time_t)((double)ep->degr_csize / degr_kps); + degr_time = (time_t)((double)ep->degr_est->csize / degr_kps); } } else { dump_kps = fix_rate(ep->incrrate); - dump_time = (time_t)((double)ep->dump_csize / dump_kps); + dump_time = (time_t)((double)ep->dump_est->csize / dump_kps); } - if(ep->dump_level == 0 && ep->degr_csize != (gint64)-1) { + if(ep->dump_est->level == 0 && ep->degr_est->csize != (gint64)-1) { g_snprintf(degr_level_str, sizeof(degr_level_str), - "%d", ep->degr_level); + "%d", ep->degr_est->level); g_snprintf(degr_nsize_str, sizeof(degr_nsize_str), - "%lld", (long long)ep->degr_nsize); + "%lld", (long long)ep->degr_est->nsize); g_snprintf(degr_csize_str, sizeof(degr_csize_str), - "%lld", (long long)ep->degr_csize); + "%lld", (long long)ep->degr_est->csize); g_snprintf(degr_time_str, sizeof(degr_time_str), "%lld", (long long)degr_time); g_snprintf(degr_kps_str, sizeof(degr_kps_str), @@ -3095,7 +3188,7 @@ static void output_scheduleline( if (ep->degr_mesg) { degr_mesg = quote_string(ep->degr_mesg); } else { - degr_mesg = quote_string(_("Can't switch to degraded mode for unknown reason")); + degr_mesg = quote_string(_("Skipping: cannot dump in degraded mode for unknown reason")); } degr_str = vstralloc(" ", degr_mesg, NULL); amfree(degr_mesg); @@ -3103,11 +3196,11 @@ static void output_scheduleline( g_snprintf(dump_priority_str, SIZEOF(dump_priority_str), "%d", ep->dump_priority); g_snprintf(dump_level_str, SIZEOF(dump_level_str), - "%d", ep->dump_level); + "%d", ep->dump_est->level); g_snprintf(dump_nsize_str, sizeof(dump_nsize_str), - "%lld", (long long)ep->dump_nsize); + "%lld", (long long)ep->dump_est->nsize); g_snprintf(dump_csize_str, sizeof(dump_csize_str), - "%lld", (long long)ep->dump_csize); + "%lld", (long long)ep->dump_est->csize); g_snprintf(dump_time_str, sizeof(dump_time_str), "%lld", (long long)dump_time); g_snprintf(dump_kps_str, sizeof(dump_kps_str), @@ -3127,6 +3220,9 @@ static void output_scheduleline( degr_str ? degr_str : "", "\n", NULL); + if (est(dp)->dump_est->guessed == 1) { + log_add(L_WARNING, _("WARNING: no history available for %s:%s; guessing that size will be %lld KB\n"), dp->host->hostname, qname, (long long)est(dp)->dump_est->csize); + } fputs(schedline, stdout); fputs(schedline, stderr); amfree(features); @@ -3134,3 +3230,22 @@ static void output_scheduleline( amfree(degr_str); amfree(qname); } + +static void +server_estimate( + disk_t *dp, + int i, + info_t *info, + int level) +{ + int stats; + gint64 size; + + size = internal_server_estimate(dp, info, level, &stats); + + est(dp)->dump_est = &est(dp)->estimate[i]; + est(dp)->estimate[i].nsize = size; + if (stats == 0) { + est(dp)->estimate[i].guessed = 1; + } +}