2 # Copyright (c) 2008, 2009, 2010 Zmanda, Inc. All Rights Reserved.
4 # This program is free software; you can redistribute it and/or modify it
5 # under the terms of the GNU General Public License version 2 as published
6 # by the Free Software Foundation.
8 # This program is distributed in the hope that it will be useful, but
9 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 # Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
18 # Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
20 # This is a tool to examine a device and generate a reasonable tapetype
23 use lib '@amperldir@';
29 use Amanda::BigIntCompat;
31 use Amanda::Device qw( :constants );
32 use Amanda::Debug qw( :logging );
33 use Amanda::Util qw( :constants );
34 use Amanda::Config qw( :init :getconf config_dir_relative );
37 use Amanda::Constants;
40 # command-line options
41 my $opt_only_compression = 0;
43 my $opt_tapetype_name = 'unknown-tapetype';
45 my $opt_label = "amtapetype-".(int rand 2**31);
50 # global "hint" from the compression heuristic as to how fast this
52 my $device_speed_estimate;
54 # open up a device, optionally check its label on the first invocation,
55 # and start it in ACCESS_WRITE.
56 my $_label_checked = 0;
58 my $device = Amanda::Device->new($opt_device_name);
59 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
60 die("Could not open device $opt_device_name: ".$device->error()."\n");
63 if (!$device->configure(0)) {
64 die("Errors configuring $opt_device_name: " . $device->error_or_status());
67 if (defined $opt_blocksize) {
68 $device->property_set('BLOCK_SIZE', $opt_blocksize)
69 or die "Error setting blocksize: " . $device->error_or_status();
72 if (!$opt_force and !$_label_checked) {
73 my $read_label_status = $device->read_label();
74 if ($read_label_status & $DEVICE_STATUS_VOLUME_UNLABELED) {
76 } elsif ($read_label_status != $DEVICE_STATUS_SUCCESS) {
77 die "Error reading label: " . $device->error_or_status();
78 } elsif ($device->volume_label) {
79 die "Volume in device $opt_device_name has Amanda label '" .
80 $device->volume_label . "'. Giving up.";
85 my $start_time = time;
89 last if ($device->start($ACCESS_WRITE, $opt_label, undef));
90 if (!($device->status & $DEVICE_STATUS_DEVICE_BUSY)) {
91 die("Error writing label '$opt_label': ". $device->error_or_status());
95 print STDERR "Device is busy. Amtapetype will retry forever; hit ctrl-C to quit.\n";
100 $backoff = 120 if $backoff > 120;
105 my $elapsed = time - $start_time;
106 print STDERR "Drive was busy for $elapsed seconds.\n";
107 print STDERR "If this device is used in a changer, you may want to set timeouts appropriately.\n";
113 # Write a single file to the device, and record the results in STATS.
115 # STATS => $stats_hashref, (see below)
116 # DEVICE => $dev, (device to write to)
117 # PATTERN => RANDOM or FIXED, (data pattern to write)
118 # BYTES => nn, (number of bytes; optional)
119 # MAX_TIME => secs); (cancel write after this time; optional)
121 # Returns 0 on success (including EOM), "TIMEOUT" on timeout, or an error message
124 # STATS is a multi-level hashref; write_one_file adds to any values
125 # already in the data structure.
126 # $stats->{$pattern}->{TIME} - number of seconds spent writing
127 # $stats->{$pattern}->{FILES} - number of files written
128 # $stats->{$pattern}->{BYTES} - number of bytes written (approximate)
130 sub write_one_file(%) {
132 my $stats = $options{'STATS'} || { };
133 my $device = $options{'DEVICE'};
134 my $bytes = $options{'MAX_BYTES'} || 0;
135 my $pattern = $options{'PATTERN'} || 'FIXED';
136 my $max_time = $options{'MAX_TIME'} || 0;
138 # get the block size now, while the device is still working
139 my $block_size = $device->property_get("block_size");
142 my $hdr = Amanda::Header->new();
143 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
144 $hdr->{name} = "amtapetype";
145 $hdr->{disk} = "/test";
146 $hdr->{datestamp} = "X";
147 $hdr->{program} = "AMTAPETYPE";
148 $device->start_file($hdr)
149 or return $device->error_or_status();
151 # set up the transfer
152 my ($source, $dest, $xfer);
153 if ($pattern eq 'FIXED') {
154 # a simple 256-byte pattern to dodge run length encoding.
155 my $non_random_pattern = pack("C*", 0..255);
156 $source = Amanda::Xfer::Source::Pattern->new($bytes, $non_random_pattern);
157 } elsif ($pattern eq 'RANDOM') {
158 $source = Amanda::Xfer::Source::Random->new($bytes, 1 + int rand 100);
160 die "Unknown PATTERN $pattern";
162 $dest = Amanda::Xfer::Dest::Device->new($device, 0);
163 $xfer = Amanda::Xfer->new([$source, $dest]);
165 # set up the relevant callbacks
166 my ($timeout_src, $spinner_src);
171 $timeout_src = Amanda::MainLoop::timeout_source($max_time * 1000);
172 $timeout_src->set_callback(sub {
175 $xfer->cancel(); # will result in an XFER_DONE
179 $spinner_src = Amanda::MainLoop::timeout_source(1000);
180 $spinner_src->set_callback(sub {
182 my ($file, $block) = ($device->file(), $device->block());
183 print STDERR "File $file, block $block \r";
186 my $start_time = time();
189 my ($src, $xmsg, $xfer) = @_;
190 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
191 $got_error = $xmsg->{message};
192 } elsif ($xmsg->{'type'} == $Amanda::Xfer::XMSG_DONE) {
193 Amanda::MainLoop::quit();
197 Amanda::MainLoop::run();
198 $spinner_src->remove();
199 $timeout_src->remove() if ($timeout_src);
200 print STDERR " " x 60, "\r";
202 my $duration = time() - $start_time;
204 # OK, we finished, update statistics (even if we saw an error)
205 my $blocks_written = $device->block();
206 $stats->{$pattern}->{BYTES} += $blocks_written * $block_size;
207 $stats->{$pattern}->{FILES} += 1;
208 $stats->{$pattern}->{TIME} += $duration;
210 # make sure the time is nonzero
211 if ($stats->{$pattern}->{TIME} == 0) {
212 $stats->{$pattern}->{TIME}++;
215 if ($device->status() != $Amanda::Device::DEVICE_STATUS_SUCCESS) {
216 return $device->error_or_status();
230 sub check_compression {
231 my $device = open_device();
233 # Check compression status here by property query. If the device can answer
234 # the question, there's no reason to investigate further.
235 my $compression_enabled = $device->property_get("compression");
237 if (defined $compression_enabled) {
238 return $compression_enabled;
241 # Need to use heuristic to find out if compression is enabled. Also, we
242 # rewind between passes so that the second pass doesn't get some kind of
243 # buffering advantage.
245 print STDERR "Applying heuristic check for compression.\n";
247 # We base our determination on whether it's faster to write random data or
248 # patterned data. That starts by writing random data for a short length of
249 # time, then measuring the elapsed time and total data written. Due to
250 # potential delay in cancelling a transfer, the elapsed time will be a bit
251 # longer than the intended time. We then write the same amount of
252 # patterned data, and again measure the elapsed time. We can then
253 # calculate the speeds of the two operations. If the compressible speed
254 # was faster by more than min_ratio, then we assume compression is enabled.
256 my $compression_check_time = 60;
257 my $compression_check_min_ratio = 1.2;
261 my $err = write_one_file(
264 MAX_TIME => $compression_check_time,
265 PATTERN => 'RANDOM');
267 if ($err != 'TIMEOUT') {
272 # speed calculations are a little tricky: BigInt * float comes out to NaN, so we
273 # cast the BigInts to float first
274 my $random_speed = ($stats->{RANDOM}->{BYTES} . "") / $stats->{RANDOM}->{TIME};
275 print STDERR "Wrote random (uncompressible) data at $random_speed bytes/sec\n";
277 # sock this away for make_tapetype's use
278 $device_speed_estimate = $random_speed;
280 # restart the device to clear any errors and rewind it
281 $device = open_device();
283 $err = write_one_file(
286 MAX_BYTES => $stats->{'RANDOM'}->{'BYTES'},
293 my $fixed_speed = ($stats->{FIXED}->{BYTES} . "") / $stats->{FIXED}->{TIME};
294 print STDERR "Wrote fixed (compressible) data at $fixed_speed bytes/sec\n";
296 $compression_enabled =
297 ($fixed_speed / $random_speed > $compression_check_min_ratio);
298 return $compression_enabled;
305 my $xfer = Amanda::Xfer->new([
306 Amanda::Xfer::Source::Device->new($device),
307 Amanda::Xfer::Dest::Null->new(0),
311 my ($src, $xmsg, $xfer) = @_;
312 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
313 $got_error = $xmsg->{message};
314 } elsif ($xmsg->{'type'} == $Amanda::Xfer::XMSG_DONE) {
315 Amanda::MainLoop::quit();
319 Amanda::MainLoop::run();
323 my $device = open_device();
325 print STDERR "Checking for FSF_AFTER_FILEMARK requirement\n";
326 my $fsf_after_filemark = $device->property_get("FSF_AFTER_FILEMARK");
328 # not a 'tape:' device
329 return if !defined $fsf_after_filemark;
331 $device->start($ACCESS_WRITE, "TEST-001", "20080706050403");
333 my $hdr = new Amanda::Header;
335 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
336 $hdr->{name} = "localhost";
337 $hdr->{disk} = "/test1";
338 $hdr->{datestamp} = "20080706050403";
339 $hdr->{program} = "AMTAPETYPE";
340 $device->start_file($hdr);
341 $device->finish_file();
343 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
344 $hdr->{name} = "localhost";
345 $hdr->{disk} = "/test2";
346 $hdr->{datestamp} = "20080706050403";
347 $hdr->{program} = "AMTAPETYPE";
348 $device->start_file($hdr);
349 $device->finish_file();
351 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
352 $hdr->{name} = "localhost";
353 $hdr->{disk} = "/test3";
354 $hdr->{datestamp} = "20080706050403";
355 $hdr->{program} = "AMTAPETYPE";
356 $device->start_file($hdr);
357 $device->finish_file();
361 #set fsf_after_filemark to false
362 $device->property_set('FSF_AFTER_FILEMARK', 0)
363 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
365 my $need_fsf_after_filemark = 0;
367 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
368 die ("Could not read label from: " . $device->error_or_status());
370 if ($device->volume_label != "TEST-001") {
371 die ("wrong label: ", $device->volume_label);
373 $device->start($ACCESS_READ, undef, undef)
374 or die ("Could not start device: " . $device->error_or_status());
376 $hdr = $device->seek_file(1);
377 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
378 die ("seek_file(1) failed");
380 if ($hdr->{disk} ne "/test1") {
381 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
383 data_to_null($device);
385 $hdr = $device->seek_file(2);
386 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
387 if ($hdr->{disk} ne "/test2") {
388 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
390 data_to_null($device);
392 $hdr = $device->seek_file(3);
393 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
394 die("seek_file(3) failed");
396 if ($hdr->{disk} ne "/test3") {
397 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
399 data_to_null($device);
403 $need_fsf_after_filemark = 1;
405 # $device is in error, so open a new one
407 $device = open_device();
410 #verify need_fsf_after_filemark
411 my $fsf_after_filemark_works = 0;
412 if ($need_fsf_after_filemark) {
413 #set fsf_after_filemark to true
414 $device->property_set('FSF_AFTER_FILEMARK', 1)
415 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
417 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
418 die ("Could not read label from: " . $device->error_or_status());
420 if ($device->volume_label != "TEST-001") {
421 die ("wrong label: ", $device->volume_label);
423 $device->start($ACCESS_READ, undef, undef)
424 or die ("Could not start device: " . $device->error_or_status());
426 $hdr = $device->seek_file(1);
427 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
428 die ("seek_file(1) failed");
430 if ($hdr->{disk} ne "/test1") {
431 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
433 data_to_null($device);
435 $hdr = $device->seek_file(2);
436 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
437 if ($hdr->{disk} ne "/test2") {
438 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
440 data_to_null($device);
442 $hdr = $device->seek_file(3);
443 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
444 die("seek_file(3) failed");
446 if ($hdr->{disk} ne "/test3") {
447 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
449 data_to_null($device);
450 $fsf_after_filemark_works = 1;
452 die("seek_file(2) failed");
457 if ($need_fsf_after_filemark == 0 && $fsf_after_filemark_works == 0) {
458 if (defined $opt_property || $fsf_after_filemark) {
459 print STDOUT "device_property \"FSF_AFTER_FILEMARK\" \"false\"\n";
461 $device->property_set('FSF_AFTER_FILEMARK', 0);
462 } elsif ($need_fsf_after_filemark == 1 && $fsf_after_filemark_works == 1) {
463 if (defined $opt_property || !$fsf_after_filemark) {
464 print STDOUT "device_property \"FSF_AFTER_FILEMARK\" \"true\"\n";
466 $device->property_set('FSF_AFTER_FILEMARK', 1);
468 die ("Broken seek_file");
471 #Check seek to file 1 from header
472 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
473 die ("Could not read label from: " . $device->error_or_status());
475 if ($device->volume_label != "TEST-001") {
476 die ("wrong label: ", $device->volume_label);
478 $device->start($ACCESS_READ, undef, undef)
479 or die ("Could not start device: " . $device->error_or_status());
481 $hdr = $device->seek_file(1);
482 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
483 die ("seek_file(1) failed");
485 if ($hdr->{disk} ne "/test1") {
486 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
490 #Check seek to file 2 from header
491 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
492 die ("Could not read label from: " . $device->error_or_status());
494 if ($device->volume_label != "TEST-001") {
495 die ("wrong label: ", $device->volume_label);
497 $device->start($ACCESS_READ, undef, undef)
498 or die ("Could not start device: " . $device->error_or_status());
500 $hdr = $device->seek_file(2);
501 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
502 die ("seek_file(2) failed");
504 if ($hdr->{disk} ne "/test2") {
505 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
509 #Check seek to file 3 from header
510 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
511 die ("Could not read label from: " . $device->error_or_status());
513 if ($device->volume_label != "TEST-001") {
514 die ("wrong label: ", $device->volume_label);
516 $device->start($ACCESS_READ, undef, undef)
517 or die ("Could not start device: " . $device->error_or_status());
519 $hdr = $device->seek_file(3);
520 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
521 die ("seek_file(3) failed");
523 if ($hdr->{disk} ne "/test3") {
524 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
528 #Check seek to file 3 from eof of 1
529 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
530 die ("Could not read label from: " . $device->error_or_status());
532 if ($device->volume_label != "TEST-001") {
533 die ("wrong label: ", $device->volume_label);
535 $device->start($ACCESS_READ, undef, undef)
536 or die ("Could not start device: " . $device->error_or_status());
538 $hdr = $device->seek_file(1);
539 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
540 die ("seek_file(1) failed");
542 data_to_null($device);
543 $hdr = $device->seek_file(3);
544 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
545 die ("seek_file(3) failed");
547 if ($hdr->{disk} ne "/test3") {
548 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
554 my ($compression_enabled) = @_;
556 my $device = open_device();
557 my $blocksize = $device->property_get("BLOCK_SIZE");
559 # First, write one very long file to get the total tape length
560 print STDERR "Writing one file to fill the volume.\n";
562 my $err = write_one_file(
565 PATTERN => 'RANDOM');
567 # if we wrote almost no data, then there's probably a problem
568 # with the device, so error out
569 if ($stats->{RANDOM}->{BYTES} < 1024 * 1024) {
570 die "Wrote less than 1MB to the device: $err\n";
572 my $volume_size_estimate = $stats->{RANDOM}->{BYTES};
573 my $speed_estimate = (($stats->{RANDOM}->{BYTES}."") / 1024)
574 / $stats->{RANDOM}->{TIME};
575 $speed_estimate = int $speed_estimate;
576 print STDERR "Wrote $volume_size_estimate bytes at $speed_estimate kb/sec\n";
578 # now we want to write about 100 filemarks; round down to the blocksize
579 # to avoid counting padding as part of the filemark
580 my $file_size = $volume_size_estimate / 100;
581 $file_size -= $file_size % $blocksize;
583 print STDERR "Writing smaller files ($file_size bytes) to determine filemark.\n";
585 $device = open_device(); # re-open to rewind and clear errors
587 while (!write_one_file(
590 MAX_BYTES => $file_size,
591 PATTERN => 'RANDOM')) { }
593 my $filemark_estimate = ($volume_size_estimate - $stats->{RANDOM}->{BYTES})
594 / ($stats->{RANDOM}->{FILES} - 1);
595 if ($filemark_estimate < 0) {
596 $filemark_estimate = 0;
599 my $comment = "Created by amtapetype; compression "
600 . ($compression_enabled? "enabled" : "disabled");
602 # round these parameters to the nearest kb, since the parameters' units
604 my $volume_size_estimate_kb = $volume_size_estimate/1024;
605 my $filemark_kb = $filemark_estimate/1024;
607 # and suggest using device_property for blocksize if it's not an even multiple
610 if ($blocksize % 1024 == 0) {
611 $blocksize_line = "blocksize " . $blocksize/1024 . " kbytes";
613 $blocksize_line = "# add device_property \"BLOCK_SIZE\" \"$blocksize\" to the device";
617 define tapetype $opt_tapetype_name {
619 length $volume_size_estimate_kb kbytes
620 filemark $filemark_kb kbytes
621 speed $speed_estimate kps
629 Usage: amtapetype [-h] [-c] [-f] [-b blocksize] [-t typename] [-l label]
630 [ [-o config_override] ... ] [config] device
631 -h Display this message
632 -c Only check hardware compression state
633 -f Run amtapetype even if the loaded volume is already in use
634 or compression is enabled.
635 -b Blocksize to use (default 32k)
636 -t Name to give to the new tapetype definition
637 -l Label to write to the tape (default is randomly generated)
638 -p Check property of the device.
639 -o Overwrite configuration parameter (such as device properties)
640 Blocksize can include an optional suffix (k, m, or g)
642 If CONFIG is specified, the device and its configuration are loaded
643 from the correspnding amanda.conf.
648 ## Application initialization
650 Amanda::Util::setup_application("amtapetype", "server", $CONTEXT_CMDLINE);
651 config_init(0, undef);
653 my $config_overrides = new_config_overrides($#ARGV+1);
655 Getopt::Long::Configure(qw(bundling));
657 'help|usage|?|h' => \&usage,
658 'c' => \$opt_only_compression,
660 my ($num, $suff) = ($_[1] =~ /^([0-9]+)\s*(.*)$/);
661 die "Invalid blocksize '$_[1]'" unless (defined $num);
662 my $mult = (defined $suff)?
663 Amanda::Config::find_multiplier($suff) : 1;
664 die "Invalid suffix '$suff'" unless ($mult);
665 $opt_blocksize = $num * $mult;
667 't=s' => \$opt_tapetype_name,
670 'p' => \$opt_property,
671 'o=s' => sub { add_config_override_opt($config_overrides, $_[1]); },
673 usage() if (@ARGV < 1 or @ARGV > 2);
675 set_config_overrides($config_overrides);
677 $opt_config = shift @ARGV;
678 config_init($CONFIG_INIT_EXPLICIT_NAME, $opt_config);
680 config_init(0, undef);
682 $opt_device_name= shift @ARGV;
684 my ($cfgerr_level, @cfgerr_errors) = config_errors();
685 if ($cfgerr_level >= $CFGERR_WARNINGS) {
686 config_print_errors();
687 if ($cfgerr_level >= $CFGERR_ERRORS) {
688 die("errors processing config file");
692 Amanda::Util::finish_setup($RUNNING_AS_ANY);
694 # Find property of the device.
697 if (!defined $opt_property) {
698 my $compression_enabled = check_compression();
699 print STDERR "Compression: ",
700 $compression_enabled? "enabled" : "disabled",
703 if ($compression_enabled and !$opt_force) {
704 print STDERR "Turn off compression or run amtapetype with the -f option\n";
708 if (!$opt_only_compression) {
709 make_tapetype($compression_enabled);
713 Amanda::Util::finish_application();