X-Git-Url: https://git.gag.com/?a=blobdiff_plain;f=application-src%2Fampgsql.pl;h=0150af1bd18331e82b4043ad2ddee9b4f6082623;hb=c6f0a88c567f8536c498f554285aed1f8150da18;hp=5c930ed323eaf099a202112a8945125de7c64474;hpb=cd0b924f27312d57bd42f6c4fae2b795139e2d0b;p=debian%2Famanda

diff --git a/application-src/ampgsql.pl b/application-src/ampgsql.pl
index 5c930ed..0150af1 100644
--- a/application-src/ampgsql.pl
+++ b/application-src/ampgsql.pl
@@ -38,7 +38,7 @@ use Amanda::Constants;
 use Amanda::Config qw( :init :getconf  config_dir_relative string_to_boolean );
 use Amanda::Debug qw( :logging );
 use Amanda::Paths;
-use Amanda::Util qw( :constants :encoding );
+use Amanda::Util qw( :constants :encoding quote_string );
 use Amanda::MainLoop qw( :GIOCondition );
 
 my $_DATA_DIR_TAR = "data_dir.tar";
@@ -90,6 +90,7 @@ sub new {
             $self->{'props'}->{$pname} = $conf_props->{$pname}->{'values'}->[0];
         }
     }
+
     # check for properties like 'foo-pg-host' where the diskname is 'foo'
     if ($self->{'args'}->{'disk'}) {
         foreach my $pname (@PROP_NAMES) {
@@ -102,6 +103,16 @@ sub new {
         }
     }
 
+    # overwrite with dumptype properties if they are set.
+    foreach my $pname (@PROP_NAMES) {
+	my $pdumpname = $pname;
+	$pdumpname =~ s/^pg-//g;
+	$self->{'props'}->{$pname} = $self->{'args'}->{$pdumpname}
+				 if defined $self->{'args'}->{$pdumpname};
+debug("prop $pname set from dumpname $pdumpname: $self->{'args'}->{$pdumpname}")
+if defined $self->{'args'}->{$pdumpname};
+    }
+
     unless ($self->{'props'}->{'psql-path'}) {
         foreach my $pre (split(/:/, $ENV{PATH})) {
             my $psql = "$pre/psql";
@@ -128,6 +139,10 @@ sub new {
         }
     }
 
+    if (!exists $self->{'props'}->{'pg-datadir'}) {
+	$self->{'props'}->{'pg-datadir'} =  $self->{'args'}->{'device'};
+    }
+
     return $self;
 }
 
@@ -207,7 +222,8 @@ sub _run_psql_command {
     push @cmd, "-p", $self->{'props'}->{'pg-port'} if ($self->{'props'}->{'pg-port'});
     push @cmd, "-U", $self->{'props'}->{'pg-user'} if ($self->{'props'}->{'pg-user'});
 
-    push @cmd, '--quiet', '--output', '/dev/null', '--command', $cmd, $self->{'props'}->{'pg-db'};
+    push @cmd, '--quiet', '--output', '/dev/null' if (!($cmd =~ /pg_xlogfile_name_offset/));
+    push @cmd, '--command', $cmd, $self->{'props'}->{'pg-db'};
     debug("running " . join(" ", @cmd));
 
     my ($wtr, $rdr);
@@ -229,7 +245,17 @@ sub _run_psql_command {
 	    return;
 	}
 	chomp $line;
+	return if $line =~ /^\s*$/;
 	debug("psql stdout: $line");
+	if ($cmd =~ /pg_xlogfile_name_offset/) {
+	    return if $line =~ /file_name/;
+	    return if $line =~ /------/;
+	    return if $line =~ /\(1 row\)/;
+	    if ($line =~ /^ ($_WAL_FILE_PAT)/) {
+		$self->{'switch_xlog_filename'} = $1;
+		return;
+	    }
+	}
 	if ($line =~ /NOTICE: pg_stop_backup complete, all required WAL segments have been archived/) {
 	} else {
 	    $self->print_to_server("psql stdout: $line",
@@ -278,6 +304,11 @@ sub command_selfcheck {
        exit(1);
    };
 
+    $self->print_to_server("disk " . quote_string($self->{args}->{disk}));
+
+    $self->print_to_server("ampgsql version " . $Amanda::Constants::VERSION,
+			   $Amanda::Script_App::GOOD);
+
     for my $k (keys %{$self->{'args'}}) {
         print "OK application property: $k = $self->{'args'}->{$k}\n";
     }
@@ -375,6 +406,20 @@ sub command_selfcheck {
 	    $try_connect = 0;
 	}
 
+	if ($try_connect) {
+	    my @pv = `$self->{'props'}->{'psql-path'} --version`;
+	    if ($? >> 8 == 0) {
+		$pv[0] =~ /^[^0-9]*([0-9.]*)[^0-9]*$/;
+		my $pv = $1;
+		$self->print_to_server("ampgsql psql-version $pv",
+				       $Amanda::Script_App::GOOD);
+	    } else {
+		$self->print_to_server(
+		"[Can't get " . $self->{'props'}->{'psql-path'} . " version]\n",
+		$Amanda::Script_App::ERROR);
+	    }
+	}
+
 	if ($try_connect) {
 	    $try_connect &&=
 		_check("Connecting to database server", "succeeded", "failed",
@@ -393,6 +438,20 @@ sub command_selfcheck {
 		       sub {my ($start, $end) = _get_backup_info($self, $label); $start and $end});
 	    }
 	}
+
+	{
+	    my @gv = `$self->{'args'}->{'gnutar-path'} --version`;
+	    if ($? >> 8 == 0) {
+		$gv[0] =~ /^[^0-9]*([0-9.]*)[^0-9]*$/;
+		my $gv = $1;
+		$self->print_to_server("ampgsql gtar-version $gv",
+				       $Amanda::Script_App::GOOD);
+	    } else {
+		$self->print_to_server(
+		"[Can't get " . $self->{'props'}->{'gnutar-path'} . " version]\n",
+		$Amanda::Script_App::ERROR);
+	    }
+	}
     }
 }
 
@@ -400,7 +459,9 @@ sub _state_filename {
     my ($self, $level) = @_;
 
     my @parts = ("ampgsql", hexencode($self->{'args'}->{'host'}), hexencode($self->{'args'}->{'disk'}), $level);
-    $self->{'args'}->{'statedir'} . '/'  . join("-", @parts);
+    my $statefile = $self->{'args'}->{'statedir'} . '/'  . join("-", @parts);
+    debug("statefile: $statefile");
+    return $statefile;
 }
 
 sub _write_state_file {
@@ -443,6 +504,15 @@ sub _get_prev_state {
     $end_wal;
 }
 
+sub _make_dummy_dir_base {
+    my ($self) = @_;
+
+   my $dummydir = "$self->{'args'}->{'tmpdir'}/ampgsql-dummy-$$";
+   mkpath("$dummydir/$_ARCHIVE_DIR_RESTORE");
+
+   return $dummydir;
+}
+
 sub _make_dummy_dir {
     my ($self) = @_;
 
@@ -457,9 +527,10 @@ sub _make_dummy_dir {
 sub _run_tar_totals {
     my ($self, @other_args) = @_;
 
-    my @cmd = ($self->{'runtar'}, $self->{'args'}->{'config'},
+    my @cmd;
+    @cmd = ($self->{'runtar'}, $self->{'args'}->{'config'},
         $Amanda::Constants::GNUTAR, '--create', '--totals', @other_args);
-    debug("running " . join(" ", @cmd));
+    debug("running: " . join(" ", @cmd));
 
     local (*TAR_IN, *TAR_OUT, *TAR_ERR);
     open TAR_OUT, ">&", $self->{'out_h'};
@@ -556,6 +627,7 @@ sub _get_backup_info {
                        $lab = $1;
                    }
                }
+	       close TAROUT;
                if ($lab and $lab eq $label) {
                    $start_wal = $start;
                    $end_wal = $end;
@@ -571,7 +643,12 @@ sub _get_backup_info {
 	   debug("$bfile named WALs $start_wal .. $end_wal");
 
            # try to cleanup a bit, although this may fail and that's ok
-           unlink("$self->{'props'}->{'pg-archivedir'}/$bfile");
+	   my $filename = "$self->{'props'}->{'pg-archivedir'}/$bfile";
+           if (unlink($filename) == 0) {
+               debug("Failed to unlink '$filename': $!");
+               $self->print_to_server("Failed to unlink '$filename': $!",
+				      $Amanda::Script_App::ERROR);
+	   }
            last;
        }
        sleep(1);
@@ -632,10 +709,10 @@ sub _wait_for_wal {
     my $stoptime = time() + $maxwait;
     while ($maxwait == 0 || time < $stoptime || $count++ < 4) {
 	return if -f "$archive_dir/$wal";
-	
+
 	# for versions 8.0 or 8.1, the only way to "force" a WAL archive is to write
 	# garbage to the database.
-	if ($pg_version < 802000) {
+	if ($pg_version < 80200) {
 	    $self->_write_garbage_to_db();
 	} else {
 	    sleep(1);
@@ -651,28 +728,10 @@ sub _base_backup {
    debug("running _base_backup");
 
    my $label = "$self->{'label-prefix'}-" . time();
-   my $tmp = "$self->{'args'}->{'tmpdir'}/$label";
 
    -d $self->{'props'}->{'pg-archivedir'} or
 	die("WAL file archive directory does not exist (or is not a directory)");
 
-   # try to protect what we create
-   my $old_umask = umask();
-   umask(077);
-
-   my $cleanup = sub {
-       umask($old_umask);
-       eval {rmtree($tmp); 1}
-   };
-   my $old_die = $self->{'die_cb'};
-   $self->{'die_cb'} = sub {
-       my $msg = shift @_;
-       $cleanup->();
-       $old_die->($msg);
-   };
-   eval {rmtree($tmp,{'keep_root' => 1}); 1} or $self->{'die_cb'}->("Failed to clear tmp directory: $@");
-   eval {mkpath($tmp, 0, 0700); 1} or $self->{'die_cb'}->("Failed to create tmp directory: $@");
-
    _run_psql_command($self, "SELECT pg_start_backup('$label')") or
        $self->{'die_cb'}->("Failed to call pg_start_backup");
 
@@ -687,10 +746,11 @@ sub _base_backup {
        }
        $old_die_cb->($msg);
    };
-   _run_tar_totals($self, '--file', "$tmp/$_DATA_DIR_TAR",
+   my $size = _run_tar_totals($self, '--file', "-",
        '--directory', $self->{'props'}->{'pg-datadir'},
        '--exclude', 'postmaster.pid',
        '--exclude', 'pg_xlog/*', # contains WAL files; will be handled below
+       '--transform', "s,^,$_DATA_DIR_RESTORE/,S",
        ".");
    $self->{'die_cb'} = $old_die_cb;
 
@@ -723,22 +783,19 @@ sub _base_backup {
    $adir->close();
 
    if (@wal_files) {
-       _run_tar_totals($self, '--file', "$tmp/$_ARCHIVE_DIR_TAR",
-	   '--directory', $self->{'props'}->{'pg-archivedir'}, @wal_files);
+       $size += _run_tar_totals($self, '--file', "-",
+	   '--directory', $self->{'props'}->{'pg-archivedir'},
+	   '--transform', "s,^,$_ARCHIVE_DIR_RESTORE/,S",
+	   @wal_files);
    } else {
-       my $dummydir = $self->_make_dummy_dir();
+       my $dummydir = $self->_make_dummy_dir_base();
        $self->{'done_cb'}->(_run_tar_totals($self, '--file', '-',
-           '--directory', $dummydir, "empty-incremental"));
+           '--directory', $dummydir, "$_ARCHIVE_DIR_RESTORE"));
        rmtree($dummydir);
    }
 
-   # create the final tar file
-   my $size = _run_tar_totals($self, '--directory', $tmp, '--file', '-',
-       $_ARCHIVE_DIR_TAR, $_DATA_DIR_TAR);
-
    $self->{'state_cb'}->($self, $end_wal);
 
-   $cleanup->();
    $self->{'done_cb'}->($size);
 }
 
@@ -747,6 +804,13 @@ sub _incr_backup {
 
    debug("running _incr_backup");
 
+   if ($self->{'action'} eq 'backup') {
+      _run_psql_command($self, "SELECT file_name from pg_xlogfile_name_offset(pg_switch_xlog())");
+      if (defined($self->{'switch_xlog_filename'})) {
+	 $self->_wait_for_wal($self->{'switch_xlog_filename'});
+      }
+   }
+
    my $end_wal = _get_prev_state($self);
    if ($end_wal) {
        debug("previously ended at: $end_wal");
@@ -820,7 +884,11 @@ sub command_backup {
        $self->{'unlink_cb'} = sub {
            my $filename = shift @_;
 	   debug("unlinking WAL file $filename");
-           unlink($filename);
+           if (unlink($filename) == 0) {
+               debug("Failed to unlink '$filename': $!");
+               $self->print_to_server("Failed to unlink '$filename': $!",
+                                      $Amanda::Script_App::ERROR);
+           }
        };
    } else {
        $self->{'unlink_cb'} = sub {
@@ -836,55 +904,82 @@ sub command_backup {
 }
 
 sub command_restore {
-   my $self = shift;
+    my $self = shift;
 
-   chdir(Amanda::Util::get_original_cwd());
-   if (defined $self->{'args'}->{directory}) {
-      if (!-d $self->{'args'}->{directory}) {
-	 $self->print_to_server_and_die("Directory $self->{directory}: $!",
-					$Amanda::Script_App::ERROR);
-      }
-      if (!-w $self->{'args'}->{directory}) {
-	 $self->print_to_server_and_die("Directory $self->{directory}: $!",
-					$Amanda::Script_App::ERROR);
-      }
-      chdir($self->{'args'}->{directory});
-   }
-   my $cur_dir = POSIX::getcwd();
+    chdir(Amanda::Util::get_original_cwd());
+    if (defined $self->{'args'}->{directory}) {
+	if (!-d $self->{'args'}->{directory}) {
+	    $self->print_to_server_and_die("Directory $self->{directory}: $!",
+					   $Amanda::Script_App::ERROR);
+	}
+	if (!-w $self->{'args'}->{directory}) {
+	    $self->print_to_server_and_die("Directory $self->{directory}: $!",
+					   $Amanda::Script_App::ERROR);
+	}
+	chdir($self->{'args'}->{directory});
+    }
+    my $cur_dir = POSIX::getcwd();
 
-   if (!-d $_ARCHIVE_DIR_RESTORE) {
-       mkdir($_ARCHIVE_DIR_RESTORE) or die("could not create archive WAL directory: $!");
-   }
-   my $status;
-   if ($self->{'args'}->{'level'} > 0) {
-       debug("extracting incremental backup to $cur_dir/$_ARCHIVE_DIR_RESTORE");
-       $status = system($self->{'args'}->{'gnutar-path'}, '--extract',
-	   '--file', '-',
-	   '--ignore-zeros',
-	   '--exclude', 'empty-incremental',
-           '--directory', $_ARCHIVE_DIR_RESTORE) >> 8;
-       (0 == $status) or die("Failed to extract level $self->{'args'}->{'level'} backup (exit status: $status)");
-   } else {
-       debug("extracting base of full backup");
-       if (!-d $_DATA_DIR_RESTORE) {
-           mkdir($_DATA_DIR_RESTORE) or die("could not create archive WAL directory: $!");
-       }
-       $status = system($self->{'args'}->{'gnutar-path'}, '--extract', '--file', '-',) >> 8;
-       (0 == $status) or die("Failed to extract base backup (exit status: $status)");
-
-       debug("extracting archive dir to $cur_dir/$_ARCHIVE_DIR_RESTORE");
-       $status = system($self->{'args'}->{'gnutar-path'}, '--extract',
-	  '--exclude', 'empty-incremental',
-          '--file', $_ARCHIVE_DIR_TAR, '--directory', $_ARCHIVE_DIR_RESTORE) >> 8;
-       (0 == $status) or die("Failed to extract archived WAL files from base backup (exit status: $status)");
-       unlink($_ARCHIVE_DIR_TAR);
-
-       debug("extracting data dir to $cur_dir/$_DATA_DIR_RESTORE");
-       $status = system($self->{'args'}->{'gnutar-path'}, '--extract',
-          '--file', $_DATA_DIR_TAR, '--directory', $_DATA_DIR_RESTORE) >> 8;
-       (0 == $status) or die("Failed to extract data directory from base backup (exit status: $status)");
-       unlink($_DATA_DIR_TAR);
-   }
+    if (!-d $_ARCHIVE_DIR_RESTORE) {
+	mkdir($_ARCHIVE_DIR_RESTORE) or die("could not create archive WAL directory: $!");
+    }
+    my $status;
+    if ($self->{'args'}->{'level'} > 0) {
+	debug("extracting incremental backup to $cur_dir/$_ARCHIVE_DIR_RESTORE");
+	$status = system($self->{'args'}->{'gnutar-path'},
+		'--extract',
+		'--file', '-',
+		'--ignore-zeros',
+		'--exclude', 'empty-incremental',
+		'--directory', $_ARCHIVE_DIR_RESTORE) >> 8;
+	(0 == $status) or die("Failed to extract level $self->{'args'}->{'level'} backup (exit status: $status)");
+    } else {
+	debug("extracting base of full backup to $cur_dir/$_DATA_DIR_RESTORE");
+	debug("extracting archive dir to $cur_dir/$_ARCHIVE_DIR_RESTORE");
+	if (!-d $_DATA_DIR_RESTORE) {
+	    mkdir($_DATA_DIR_RESTORE) or die("could not create archive WAL directory: $!");
+	}
+	my @cmd = ($self->{'args'}->{'gnutar-path'}, '--extract',
+		'--file', '-',
+		'--ignore-zero',
+		'--transform', "s,^DATA/,$_DATA_DIR_RESTORE/,S",
+		'--transform', "s,^WAL/,$_ARCHIVE_DIR_RESTORE/,S");
+	debug("run: " . join ' ',@cmd);
+	$status = system(@cmd) >> 8;
+	(0 == $status) or die("Failed to extract base backup (exit status: $status)");
+
+	if (-f $_ARCHIVE_DIR_TAR) {
+	    debug("extracting archive dir to $cur_dir/$_ARCHIVE_DIR_RESTORE");
+	    my @cmd = ($self->{'args'}->{'gnutar-path'}, '--extract',
+		'--exclude', 'empty-incremental',
+		'--file', $_ARCHIVE_DIR_TAR, '--directory',
+		$_ARCHIVE_DIR_RESTORE);
+	    debug("run: " . join ' ',@cmd);
+	    $status = system(@cmd) >> 8;
+	    (0 == $status) or die("Failed to extract archived WAL files from base backup (exit status: $status)");
+	    if (unlink($_ARCHIVE_DIR_TAR) == 0) {
+		debug("Failed to unlink '$_ARCHIVE_DIR_TAR': $!");
+		$self->print_to_server(
+				"Failed to unlink '$_ARCHIVE_DIR_TAR': $!",
+				$Amanda::Script_App::ERROR);
+	    }
+	}
+
+	if (-f $_DATA_DIR_TAR) {
+	    debug("extracting data dir to $cur_dir/$_DATA_DIR_RESTORE");
+	    my @cmd = ($self->{'args'}->{'gnutar-path'}, '--extract',
+		'--file', $_DATA_DIR_TAR,
+		'--directory', $_DATA_DIR_RESTORE);
+	    debug("run: " . join ' ',@cmd);
+	    $status = system(@cmd) >> 8;
+	    (0 == $status) or die("Failed to extract data directory from base backup (exit status: $status)");
+	    if (unlink($_DATA_DIR_TAR) == 0) {
+		debug("Failed to unlink '$_DATA_DIR_TAR': $!");
+		$self->print_to_server("Failed to unlink '$_DATA_DIR_TAR': $!",
+				$Amanda::Script_App::ERROR);
+	    }
+	}
+    }
 }
 
 sub command_validate {
@@ -926,9 +1021,11 @@ EOF
 }
 
 my $opts = {};
+my $opt_version;
 
 GetOptions(
     $opts,
+    'version' => \$opt_version,
     'config=s',
     'host=s',
     'disk=s',
@@ -946,8 +1043,22 @@ GetOptions(
     'statedir=s',
     'tmpdir=s',
     'gnutar-path=s',
+    'cleanupwal=s',
+    'archivedir=s',
+    'db=s',
+    'host=s',
+    'max-wal-wait=s',
+    'passfile=s',
+    'port=s',
+    'user=s',
+    'psql-path=s'
 ) or usage();
 
+if (defined $opt_version) {
+    print "ampgsql-" . $Amanda::Constants::VERSION , "\n";
+    exit(0);
+}
+
 my $application = Amanda::Application::ampgsql->new($opts);
 
 $application->do($ARGV[0]);