1 # Copyright (c) 2009 Zmanda, Inc. All Rights Reserved.
3 # This program is free software; you can redistribute it and/or modify it
4 # under the terms of the GNU General Public License version 2 as published
5 # by the Free Software Foundation.
7 # This program is distributed in the hope that it will be useful, but
8 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
9 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 # You should have received a copy of the GNU General Public License along
13 # with this program; if not, write to the Free Software Foundation, Inc.,
14 # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16 # Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
17 # Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
19 package Amanda::Holding;
21 use base qw( Exporter );
25 use POSIX qw( :fcntl_h );
30 use Amanda::Config qw( :getconf );
31 use Amanda::Debug qw( debug );
38 Amanda::Holding -- interface to the holding disks
47 for my $hfile (Amanda::Holding::files()) {
48 my $hdr = Amanda::Holding::get_header($hfile);
50 $size_per_host{$hdr->{'name'}} += Amanda::Holding::file_size($hfile);
53 Schematic for something like C<amflush>:
55 for my $ts (sort Amanda::Holding::get_all_timestamps()) {
59 for my $hfile (Amanda::Holding::get_files_for_flush(@to_dump)) {
71 A holding disk is a directory given in a holdingdisk definition in
74 =item Holding directory
76 A holding directory is a subdirectory of a holding disk, generally named by
77 timestamp. Note, however, that this package does not interpret holding
78 directory names as timestamps, and does not provide direct access to holding
83 A holding file describes one or more os-level files (holding file chunks) in a
84 holding directory, together representing a single dump file.
88 A holding chunk is an individual os-level file representing part of a holding
89 file. Chunks are kept small to avoid hitting filesystem size ilmits, and are
90 linked together internally by filename.
96 /data/holding <-- holding disk
97 /data/holding/20070306123456 <-- holding directory
98 /data/holding/20070306123456/raj._video_a <-- holding file and chunk
99 /data/holding/20070306123456/raj._video_a.1 <-- holding chunk
103 Holding-disk files do not have a block size, so the size of the header is fixed
104 at 32k. Rather than hard-code that value, use the constant DISK_BLOCK_BYTES
109 Note that this package assumes that a config has been loaded (see
112 These three functions provide basic access to holding disks, files, and chunks:
118 returns an list of active disks, each represented as a string. This does not
119 return holding disks which are defined in C<amanda.conf> but not used.
123 returns a list of active holding files on all disks. Note that a dump may span
124 multiple disks, so there is no use in selecting files only on certain holding
127 =item C<file_chunks($file)>
129 returns a list of chunks for the given file. Chunk filenames are always fully
134 C<Amanda::Holding> provides a few utility functions on holding files. Note
135 that these functions require fully qualified pathnames.
139 =item C<file_size($file, $ignore_headers)>
141 returns the size of the holding file I<in kilobytes>, ignoring the size of the
142 headers if C<$ignore_headers> is true.
144 =item C<file_unlink($file)>
146 unlinks (deletes) all chunks comprising C<$file>, returning true on success.
148 =item C<get_header($file)>
150 reads and returns the header (see L<Amanda::Header>) for C<$file>.
154 The remaining two functions are utilities for amflush and related tools:
158 =item C<get_all_timestamps()>
160 returns a sorted list of all timestamps with dumps in any active holding disk.
162 =item C<get_files_for_flush(@timestamps)>
164 returns a sorted list of files matching any of the supplied timestamps. Files
165 for which no DLE exists in the disklist are ignored. If no timestamps are
166 provided, then all timestamps are considered.
172 use constant DISK_BLOCK_BYTES => 32768;
174 our @EXPORT_OK = qw(dirs files file_chunks
175 get_files_for_flush get_all_datestamps
176 file_size file_unlink get_header);
185 unless (my ($year, $month, $day, $hour, $min, $sec) =
186 ($str =~ /(\d{4})(\d{2})(\d{2})(?:(\d{2})(\d{2})(\d{2}))/));
188 return 0 if ($year < 1990 || $year > 2999);
189 return 0 if ($month < 1 || $month > 12);
190 return 0 if ($day < 1 || $day > 31);
192 return 0 if (defined $hour and $hour > 23);
193 return 0 if (defined $min and $min > 60);
194 return 0 if (defined $sec and $sec > 60);
200 my ($file_fn, $verbose) = @_;
202 # walk disks, directories, and files with nested loops
203 for my $disk (disks()) {
204 my $diskh = IO::Dir->new($disk);
205 if (!defined $diskh) {
206 print $verbose "could not open holding dir '$disk': $!\n" if $verbose;
210 while (defined(my $datestr = $diskh->read())) {
211 next if $datestr eq '.' or $datestr eq '..';
213 my $dirfn = File::Spec->catfile($disk, $datestr);
215 if (!_is_datestr($datestr)) {
216 print $verbose "holding dir '$dirfn' is not a datestamp\n" if $verbose;
220 print $verbose "holding dir '$dirfn' is not a directory\n" if $verbose;
224 my $dirh = IO::Dir->new($dirfn);
225 while (defined(my $dirent = $dirh->read)) {
226 next if $dirent eq '.' or $dirent eq '..';
228 my $filename = File::Spec->catfile($disk, $datestr, $dirent);
230 print $verbose "holding file '$filename' is not a file\n" if $verbose;
234 my $hdr = get_header($filename);
235 next unless defined($hdr);
237 $file_fn->($filename, $hdr);
249 for my $hdname (@{getconf($CNF_HOLDINGDISK)}) {
250 my $cfg = lookup_holdingdisk($hdname);
251 next unless defined $cfg;
253 my $dir = holdingdisk_getconf($cfg, $HOLDING_DISKDIR);
254 next unless defined $dir;
266 my $each_file_fn = sub {
267 my ($filename, $header) = @_;
268 return if $header->{'type'} != $Amanda::Header::F_DUMPFILE;
270 push @results, $filename;
272 _walk($each_file_fn, $verbose);
281 my $each_file_fn = sub {
282 my ($filename, $header) = @_;
283 push @results, { filename => $filename, header => $header };
285 _walk($each_file_fn, $verbose);
290 sub merge_all_files {
295 for my $file (@files) {
296 $hfiles{$file->{'filename'}} = $file->{'header'};
299 foreach my $filename (keys %hfiles) {
300 next if !exists $hfiles{$filename};
301 if ($hfiles{$filename}->{'type'} == $Amanda::Header::F_DUMPFILE) {
302 push @result, {filename => $filename, header => $hfiles{$filename}};
303 my $is_tmp = ($filename =~ /\.tmp$/);
304 my $cont_filename = $filename;
305 my $cfilename = $hfiles{$cont_filename}->{'cont_filename'};
307 $cf .= ".tmp" if $is_tmp;
308 while (defined $cfilename && $cfilename ne "" && -f $cf) {
309 delete $hfiles{$cont_filename};
310 $cont_filename = $cf;
311 $cfilename = $hfiles{$cont_filename}->{'cont_filename'};
313 $cf .= ".tmp" if $is_tmp;
315 delete $hfiles{$cont_filename};
316 } elsif ($hfiles{$filename}->{'type'} != $Amanda::Header::F_CONT_DUMPFILE) {
317 push @result, {filename => $filename, header => $hfiles{$filename}};
318 delete $hfiles{$filename}
320 # do nothing for F_CONTFILE
324 foreach my $filename (keys %hfiles) {
325 next if !exists $hfiles{$filename};
326 if ($hfiles{$filename}->{'type'} == $Amanda::Header::F_CONT_DUMPFILE) {
327 push @result, {filename => $filename, header => $hfiles{$filename}};
329 delete $hfiles{$filename}
340 last unless -f $filename;
341 my $hdr = get_header($filename);
342 last unless defined($hdr);
344 push @results, $filename;
346 if ($hdr->{'cont_filename'}) {
347 $filename = $hdr->{'cont_filename'};
349 # no continuation -> we're done
357 sub file_tmp_chunks {
362 last unless -f $filename;
363 my $hdr = get_header($filename);
364 last unless defined($hdr);
366 push @results, $filename;
368 if ($hdr->{'cont_filename'}) {
369 $filename = $hdr->{'cont_filename'} . ".tmp";
371 # no continuation -> we're done
380 my ($filename) = shift;
381 my ($complete) = shift;
383 my @files = file_tmp_chunks($filename);
384 while (my $tmp_filename = pop @files) {
385 my $hdr = get_header($tmp_filename);
386 if ($hdr->{'is_partial'} == 0 and $complete == 0) {
387 $hdr->{'is_partial'} = 1;
388 write_header($tmp_filename, $hdr);
390 my $hfilename = $tmp_filename;
391 $hfilename =~ s/\.tmp$//;
392 rename $tmp_filename, $hfilename;
400 return unless -f $filename;
402 my $fd = POSIX::open($filename, O_RDONLY);
405 my $hdr_bytes = Amanda::Util::full_read($fd, DISK_BLOCK_BYTES);
407 if (length($hdr_bytes) == 0) {
408 my $hdr = Amanda::Header->new();
409 $hdr->{'type'} = $Amanda::Header::F_EMPTY;
411 } elsif (length($hdr_bytes) < DISK_BLOCK_BYTES) {
412 my $hdr = Amanda::Header->new();
413 $hdr->{'type'} = $Amanda::Header::F_UNKNOWN;
417 return Amanda::Header->from_string($hdr_bytes);
421 my $filename = shift;
424 return unless -f $filename;
425 my $fd = POSIX::open($filename, O_RDWR);
427 my $header = $hdr->to_string(DISK_BLOCK_BYTES, DISK_BLOCK_BYTES);
428 Amanda::Util::full_write($fd, $header, DISK_BLOCK_BYTES);
435 for my $chunk (file_chunks($filename)) {
436 unlink($chunk) or return 0;
445 for my $chunk (filetmp_chunks($filename)) {
446 unlink($chunk) or return 0;
453 # walk disks, directories, and files with nested loops
454 for my $disk (disks()) {
455 my $diskh = IO::Dir->new($disk);
456 next unless defined $diskh;
458 while (defined(my $datestr = $diskh->read())) {
459 next if $datestr eq '.' or $datestr eq '..';
461 my $dirfn = File::Spec->catfile($disk, $datestr);
462 next unless _is_datestr($datestr);
463 next unless -d $dirfn;
470 my ($filename, $ignore_headers) = @_;
471 my $total = Math::BigInt->new(0);
473 for my $chunk (file_chunks($filename)) {
474 my $sb = stat($chunk);
475 my $size = Math::BigInt->new($sb->size);
476 $size -= DISK_BLOCK_BYTES if $ignore_headers;
477 $size = ($size + 1023) / 1024;
485 sub get_files_for_flush {
489 my $each_file_fn = sub {
490 my ($filename, $header) = @_;
491 return if $header->{'type'} != $Amanda::Header::F_DUMPFILE;
493 if (@dateargs && !grep { $_ eq $header->{'datestamp'}; } @dateargs) {
497 if (!Amanda::Disklist::get_disk($header->{'name'}, $header->{'disk'})) {
501 push @results, $filename;
503 _walk($each_file_fn, 0);
505 return sort @results;
508 sub get_all_datestamps {
511 my $each_file_fn = sub {
512 my ($filename, $header) = @_;
513 return if $header->{'type'} != $Amanda::Header::F_DUMPFILE;
515 $datestamps{$header->{'datestamp'}} = 1;
517 _walk($each_file_fn, 0);
519 return sort keys %datestamps;