2 # Copyright (c) 2008, 2009, 2010 Zmanda, Inc. All Rights Reserved.
4 # This program is free software; you can redistribute it and/or modify it
5 # under the terms of the GNU General Public License version 2 as published
6 # by the Free Software Foundation.
8 # This program is distributed in the hope that it will be useful, but
9 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 # Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
18 # Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
20 # This is a tool to examine a device and generate a reasonable tapetype
23 use lib '@amperldir@';
30 use Amanda::BigIntCompat;
32 use Amanda::Device qw( :constants );
33 use Amanda::Debug qw( :logging );
34 use Amanda::Util qw( :constants );
35 use Amanda::Config qw( :init :getconf config_dir_relative );
38 use Amanda::Constants;
41 # command-line options
42 my $opt_only_compression = 0;
44 my $opt_tapetype_name = 'unknown-tapetype';
46 my $opt_label = "amtapetype-".(int rand 2**31);
51 # global "hint" from the compression heuristic as to how fast this
53 my $device_speed_estimate;
55 # open up a device, optionally check its label on the first invocation,
56 # and start it in ACCESS_WRITE.
57 my $_label_checked = 0;
59 my $device = Amanda::Device->new($opt_device_name);
60 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
61 die("Could not open device $opt_device_name: ".$device->error()."\n");
64 if (!$device->configure(0)) {
65 die("Errors configuring $opt_device_name: " . $device->error_or_status());
68 if (defined $opt_blocksize) {
69 $device->property_set('BLOCK_SIZE', $opt_blocksize)
70 or die "Error setting blocksize: " . $device->error_or_status();
73 if (!$opt_force and !$_label_checked) {
74 my $read_label_status = $device->read_label();
75 if ($read_label_status & $DEVICE_STATUS_VOLUME_UNLABELED) {
77 } elsif ($read_label_status != $DEVICE_STATUS_SUCCESS) {
78 die "Error reading label: " . $device->error_or_status();
79 } elsif ($device->volume_label) {
80 die "Volume in device $opt_device_name has Amanda label '" .
81 $device->volume_label . "'. Giving up.";
86 my $start_time = time;
90 last if ($device->start($ACCESS_WRITE, $opt_label, undef));
91 if (!($device->status & $DEVICE_STATUS_DEVICE_BUSY)) {
92 die("Error writing label '$opt_label': ". $device->error_or_status());
96 print STDERR "Device is busy. Amtapetype will retry forever; hit ctrl-C to quit.\n";
101 $backoff = 120 if $backoff > 120;
106 my $elapsed = time - $start_time;
107 print STDERR "Drive was busy for $elapsed seconds.\n";
108 print STDERR "If this device is used in a changer, you may want to set timeouts appropriately.\n";
114 # Write a single file to the device, and record the results in STATS.
116 # STATS => $stats_hashref, (see below)
117 # DEVICE => $dev, (device to write to)
118 # PATTERN => RANDOM or FIXED, (data pattern to write)
119 # BYTES => nn, (number of bytes; optional)
120 # MAX_TIME => secs); (cancel write after this time; optional)
122 # Returns 0 on success (including EOM), "TIMEOUT" on timeout, or an error message
125 # STATS is a multi-level hashref; write_one_file adds to any values
126 # already in the data structure.
127 # $stats->{$pattern}->{TIME} - number of seconds spent writing
128 # $stats->{$pattern}->{FILES} - number of files written
129 # $stats->{$pattern}->{BYTES} - number of bytes written (approximate)
131 sub write_one_file(%) {
133 my $stats = $options{'STATS'} || { };
134 my $device = $options{'DEVICE'};
135 my $bytes = $options{'MAX_BYTES'} || 0;
136 my $pattern = $options{'PATTERN'} || 'FIXED';
137 my $max_time = $options{'MAX_TIME'} || 0;
139 # get the block size now, while the device is still working
140 my $block_size = $device->property_get("block_size");
143 my $hdr = Amanda::Header->new();
144 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
145 $hdr->{name} = "amtapetype";
146 $hdr->{disk} = "/test";
147 $hdr->{datestamp} = "X";
148 $hdr->{program} = "AMTAPETYPE";
149 $device->start_file($hdr)
150 or return $device->error_or_status();
152 # set up the transfer
153 my ($source, $dest, $xfer);
154 if ($pattern eq 'FIXED') {
155 # a simple 256-byte pattern to dodge run length encoding.
156 my $non_random_pattern = pack("C*", 0..255);
157 $source = Amanda::Xfer::Source::Pattern->new($bytes, $non_random_pattern);
158 } elsif ($pattern eq 'RANDOM') {
159 $source = Amanda::Xfer::Source::Random->new($bytes, 1 + int rand 100);
161 die "Unknown PATTERN $pattern";
163 $dest = Amanda::Xfer::Dest::Device->new($device, 1);
164 $xfer = Amanda::Xfer->new([$source, $dest]);
166 # set up the relevant callbacks
167 my ($timeout_src, $spinner_src);
172 $timeout_src = Amanda::MainLoop::timeout_source($max_time * 1000);
173 $timeout_src->set_callback(sub {
176 $xfer->cancel(); # will result in an XFER_DONE
180 $spinner_src = Amanda::MainLoop::timeout_source(1000);
181 $spinner_src->set_callback(sub {
183 my ($file, $block) = ($device->file(), $device->block());
184 print STDERR "File $file, block $block \r";
187 my $start_time = time();
190 my ($src, $xmsg, $xfer) = @_;
191 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
192 $got_error = $xmsg->{message};
193 } elsif ($xmsg->{'type'} == $Amanda::Xfer::XMSG_DONE) {
194 Amanda::MainLoop::quit();
198 Amanda::MainLoop::run();
199 $spinner_src->remove();
200 $timeout_src->remove() if ($timeout_src);
201 print STDERR " " x 60, "\r";
203 my $duration = time() - $start_time;
205 # OK, we finished, update statistics (even if we saw an error)
206 my $blocks_written = $device->block();
207 $stats->{$pattern}->{BYTES} += $blocks_written * $block_size;
208 $stats->{$pattern}->{FILES} += 1;
209 $stats->{$pattern}->{TIME} += $duration;
211 # make sure the time is nonzero
212 if ($stats->{$pattern}->{TIME} == 0) {
213 $stats->{$pattern}->{TIME}++;
216 if ($device->status() != $Amanda::Device::DEVICE_STATUS_SUCCESS) {
217 return $device->error_or_status();
220 if ($got_error && $got_error =~ /LEOM detected/) {
235 sub check_compression {
236 my $device = open_device();
238 # Check compression status here by property query. If the device can answer
239 # the question, there's no reason to investigate further.
240 my $compression_enabled = $device->property_get("compression");
242 if (defined $compression_enabled) {
243 return $compression_enabled;
246 # Need to use heuristic to find out if compression is enabled. Also, we
247 # rewind between passes so that the second pass doesn't get some kind of
248 # buffering advantage.
250 print STDERR "Applying heuristic check for compression.\n";
252 # We base our determination on whether it's faster to write random data or
253 # patterned data. That starts by writing random data for a short length of
254 # time, then measuring the elapsed time and total data written. Due to
255 # potential delay in cancelling a transfer, the elapsed time will be a bit
256 # longer than the intended time. We then write the same amount of
257 # patterned data, and again measure the elapsed time. We can then
258 # calculate the speeds of the two operations. If the compressible speed
259 # was faster by more than min_ratio, then we assume compression is enabled.
261 my $compression_check_time = 60;
262 my $compression_check_min_ratio = 1.2;
266 my $err = write_one_file(
269 MAX_TIME => $compression_check_time,
270 PATTERN => 'RANDOM');
272 if ($err != 'TIMEOUT') {
277 # speed calculations are a little tricky: BigInt * float comes out to NaN, so we
278 # cast the BigInts to float first
279 my $random_speed = ($stats->{RANDOM}->{BYTES} . "") / $stats->{RANDOM}->{TIME};
280 print STDERR "Wrote random (uncompressible) data at $random_speed bytes/sec\n";
282 # sock this away for make_tapetype's use
283 $device_speed_estimate = $random_speed;
285 # restart the device to clear any errors and rewind it
286 $device = open_device();
288 $err = write_one_file(
291 MAX_BYTES => $stats->{'RANDOM'}->{'BYTES'},
298 my $fixed_speed = ($stats->{FIXED}->{BYTES} . "") / $stats->{FIXED}->{TIME};
299 print STDERR "Wrote fixed (compressible) data at $fixed_speed bytes/sec\n";
301 $compression_enabled =
302 ($fixed_speed / $random_speed > $compression_check_min_ratio);
303 return $compression_enabled;
310 my $xfer = Amanda::Xfer->new([
311 Amanda::Xfer::Source::Device->new($device),
312 Amanda::Xfer::Dest::Null->new(0),
316 my ($src, $xmsg, $xfer) = @_;
317 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
318 $got_error = $xmsg->{message};
319 } elsif ($xmsg->{'type'} == $Amanda::Xfer::XMSG_DONE) {
320 Amanda::MainLoop::quit();
324 Amanda::MainLoop::run();
328 my $device = open_device();
330 print STDERR "Checking for FSF_AFTER_FILEMARK requirement\n";
331 my $fsf_after_filemark = $device->property_get("FSF_AFTER_FILEMARK");
333 # not a 'tape:' device
334 return if !defined $fsf_after_filemark;
336 $device->start($ACCESS_WRITE, "TEST-001", "20080706050403");
338 my $hdr = new Amanda::Header;
340 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
341 $hdr->{name} = "localhost";
342 $hdr->{disk} = "/test1";
343 $hdr->{datestamp} = "20080706050403";
344 $hdr->{program} = "AMTAPETYPE";
345 $device->start_file($hdr);
346 $device->finish_file();
348 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
349 $hdr->{name} = "localhost";
350 $hdr->{disk} = "/test2";
351 $hdr->{datestamp} = "20080706050403";
352 $hdr->{program} = "AMTAPETYPE";
353 $device->start_file($hdr);
354 $device->finish_file();
356 $hdr->{type} = $Amanda::Header::F_DUMPFILE;
357 $hdr->{name} = "localhost";
358 $hdr->{disk} = "/test3";
359 $hdr->{datestamp} = "20080706050403";
360 $hdr->{program} = "AMTAPETYPE";
361 $device->start_file($hdr);
362 $device->finish_file();
366 #set fsf_after_filemark to false
367 $device->property_set('FSF_AFTER_FILEMARK', 0)
368 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
370 my $need_fsf_after_filemark = 0;
372 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
373 die ("Could not read label from: " . $device->error_or_status());
375 if ($device->volume_label != "TEST-001") {
376 die ("wrong label: ", $device->volume_label);
378 $device->start($ACCESS_READ, undef, undef)
379 or die ("Could not start device: " . $device->error_or_status());
381 $hdr = $device->seek_file(1);
382 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
383 die ("seek_file(1) failed");
385 if ($hdr->{disk} ne "/test1") {
386 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
388 data_to_null($device);
390 $hdr = $device->seek_file(2);
391 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
392 if ($hdr->{disk} ne "/test2") {
393 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
395 data_to_null($device);
397 $hdr = $device->seek_file(3);
398 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
399 die("seek_file(3) failed");
401 if ($hdr->{disk} ne "/test3") {
402 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
404 data_to_null($device);
408 $need_fsf_after_filemark = 1;
410 # $device is in error, so open a new one
412 $device = open_device();
415 #verify need_fsf_after_filemark
416 my $fsf_after_filemark_works = 0;
417 if ($need_fsf_after_filemark) {
418 #set fsf_after_filemark to true
419 $device->property_set('FSF_AFTER_FILEMARK', 1)
420 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
422 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
423 die ("Could not read label from: " . $device->error_or_status());
425 if ($device->volume_label != "TEST-001") {
426 die ("wrong label: ", $device->volume_label);
428 $device->start($ACCESS_READ, undef, undef)
429 or die ("Could not start device: " . $device->error_or_status());
431 $hdr = $device->seek_file(1);
432 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
433 die ("seek_file(1) failed");
435 if ($hdr->{disk} ne "/test1") {
436 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
438 data_to_null($device);
440 $hdr = $device->seek_file(2);
441 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
442 if ($hdr->{disk} ne "/test2") {
443 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
445 data_to_null($device);
447 $hdr = $device->seek_file(3);
448 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
449 die("seek_file(3) failed");
451 if ($hdr->{disk} ne "/test3") {
452 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
454 data_to_null($device);
455 $fsf_after_filemark_works = 1;
457 die("seek_file(2) failed");
462 if ($need_fsf_after_filemark == 0 && $fsf_after_filemark_works == 0) {
463 if (defined $opt_property || $fsf_after_filemark) {
464 print STDOUT "device-property \"FSF_AFTER_FILEMARK\" \"false\"\n";
466 $device->property_set('FSF_AFTER_FILEMARK', 0);
467 } elsif ($need_fsf_after_filemark == 1 && $fsf_after_filemark_works == 1) {
468 if (defined $opt_property || !$fsf_after_filemark) {
469 print STDOUT "device-property \"FSF_AFTER_FILEMARK\" \"true\"\n";
471 $device->property_set('FSF_AFTER_FILEMARK', 1);
473 die ("Broken seek_file");
476 #Check seek to file 1 from header
477 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
478 die ("Could not read label from: " . $device->error_or_status());
480 if ($device->volume_label != "TEST-001") {
481 die ("wrong label: ", $device->volume_label);
483 $device->start($ACCESS_READ, undef, undef)
484 or die ("Could not start device: " . $device->error_or_status());
486 $hdr = $device->seek_file(1);
487 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
488 die ("seek_file(1) failed");
490 if ($hdr->{disk} ne "/test1") {
491 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
495 #Check seek to file 2 from header
496 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
497 die ("Could not read label from: " . $device->error_or_status());
499 if ($device->volume_label != "TEST-001") {
500 die ("wrong label: ", $device->volume_label);
502 $device->start($ACCESS_READ, undef, undef)
503 or die ("Could not start device: " . $device->error_or_status());
505 $hdr = $device->seek_file(2);
506 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
507 die ("seek_file(2) failed");
509 if ($hdr->{disk} ne "/test2") {
510 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
514 #Check seek to file 3 from header
515 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
516 die ("Could not read label from: " . $device->error_or_status());
518 if ($device->volume_label != "TEST-001") {
519 die ("wrong label: ", $device->volume_label);
521 $device->start($ACCESS_READ, undef, undef)
522 or die ("Could not start device: " . $device->error_or_status());
524 $hdr = $device->seek_file(3);
525 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
526 die ("seek_file(3) failed");
528 if ($hdr->{disk} ne "/test3") {
529 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
533 #Check seek to file 3 from eof of 1
534 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
535 die ("Could not read label from: " . $device->error_or_status());
537 if ($device->volume_label != "TEST-001") {
538 die ("wrong label: ", $device->volume_label);
540 $device->start($ACCESS_READ, undef, undef)
541 or die ("Could not start device: " . $device->error_or_status());
543 $hdr = $device->seek_file(1);
544 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
545 die ("seek_file(1) failed");
547 data_to_null($device);
548 $hdr = $device->seek_file(3);
549 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
550 die ("seek_file(3) failed");
552 if ($hdr->{disk} ne "/test3") {
553 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
559 my ($compression_enabled) = @_;
561 my $device = open_device();
562 my $blocksize = $device->property_get("BLOCK_SIZE");
564 # First, write one very long file to get the total tape length
565 print STDERR "Writing one file to fill the volume.\n";
567 my $err = write_one_file(
570 PATTERN => 'RANDOM');
572 # if we wrote almost no data, then there's probably a problem
573 # with the device, so error out
574 if ($stats->{RANDOM}->{BYTES} < 1024 * 1024) {
575 die "Wrote less than 1MB to the device: $err\n";
577 my $volume_size_estimate = $stats->{RANDOM}->{BYTES};
578 my $speed_estimate = (($stats->{RANDOM}->{BYTES}."") / 1024)
579 / $stats->{RANDOM}->{TIME};
580 $speed_estimate = int $speed_estimate;
581 print STDERR "Wrote $volume_size_estimate bytes at $speed_estimate kb/sec\n";
584 if ($err eq 'LEOM') {
585 print STDERR "Got LEOM indication, so drive and kernel together support LEOM\n";
589 # now we want to write about 100 filemarks; round down to the blocksize
590 # to avoid counting padding as part of the filemark
591 my $file_size = $volume_size_estimate / 100;
592 $file_size -= $file_size % $blocksize;
594 print STDERR "Writing smaller files ($file_size bytes) to determine filemark.\n";
596 $device = open_device(); # re-open to rewind and clear errors
598 while (!write_one_file(
601 MAX_BYTES => $file_size,
602 PATTERN => 'RANDOM')) { }
604 my $filemark_estimate = ($volume_size_estimate - $stats->{RANDOM}->{BYTES})
605 / ($stats->{RANDOM}->{FILES} - 1);
606 if ($filemark_estimate < 0) {
607 $filemark_estimate = 0;
610 my $comment = "Created by amtapetype; compression "
611 . ($compression_enabled? "enabled" : "disabled");
613 # round these parameters to the nearest kb, since the parameters' units
615 my $volume_size_estimate_kb = $volume_size_estimate/1024;
616 my $filemark_kb = $filemark_estimate/1024;
618 # and suggest using device-property for blocksize if it's not an even multiple
621 if ($blocksize % 1024 == 0) {
622 $blocksize_line = "blocksize " . $blocksize/1024 . " kbytes";
624 $blocksize_line = "# add device-property \"BLOCK_SIZE\" \"$blocksize\" to the device";
628 define tapetype $opt_tapetype_name {
630 length $volume_size_estimate_kb kbytes
631 filemark $filemark_kb kbytes
632 speed $speed_estimate kps
638 print "# for this drive and kernel, LEOM is supported; add\n";
639 print "# device-property \"LEOM\" \"TRUE\"\n";
640 print "# for this device.\n";
642 print "# LEOM is not supported for this drive and kernel\n";
648 Usage: amtapetype [-h] [-c] [-f] [-b blocksize] [-t typename] [-l label]
649 [ [-o config_override] ... ] [config] device
650 -h Display this message
651 -c Only check hardware compression state
652 -f Run amtapetype even if the loaded volume is already in use
653 or compression is enabled.
654 -b Blocksize to use (default 32k)
655 -t Name to give to the new tapetype definition
656 -l Label to write to the tape (default is randomly generated)
657 -p Check property of the device.
658 -o Overwrite configuration parameter (such as device properties)
659 Blocksize can include an optional suffix (k, m, or g)
661 If CONFIG is specified, the device and its configuration are loaded
662 from the correspnding amanda.conf.
667 ## Application initialization
669 Amanda::Util::setup_application("amtapetype", "server", $CONTEXT_CMDLINE);
670 config_init(0, undef);
672 my $config_overrides = new_config_overrides($#ARGV+1);
674 Getopt::Long::Configure(qw(bundling));
676 'help|usage|?|h' => \&usage,
677 'c' => \$opt_only_compression,
679 my ($num, $suff) = ($_[1] =~ /^([0-9]+)\s*(.*)$/);
680 die "Invalid blocksize '$_[1]'" unless (defined $num);
681 my $mult = (defined $suff)?
682 Amanda::Config::find_multiplier($suff) : 1;
683 die "Invalid suffix '$suff'" unless ($mult);
684 $opt_blocksize = $num * $mult;
686 't=s' => \$opt_tapetype_name,
689 'p' => \$opt_property,
690 'o=s' => sub { add_config_override_opt($config_overrides, $_[1]); },
692 usage() if (@ARGV < 1 or @ARGV > 2);
694 set_config_overrides($config_overrides);
696 $opt_config = shift @ARGV;
697 config_init($CONFIG_INIT_EXPLICIT_NAME, $opt_config);
699 config_init(0, undef);
701 $opt_device_name= shift @ARGV;
703 my ($cfgerr_level, @cfgerr_errors) = config_errors();
704 if ($cfgerr_level >= $CFGERR_WARNINGS) {
705 config_print_errors();
706 if ($cfgerr_level >= $CFGERR_ERRORS) {
707 die("errors processing config file");
711 Amanda::Util::finish_setup($RUNNING_AS_ANY);
713 # Find property of the device.
716 if (!defined $opt_property) {
717 my $compression_enabled = check_compression();
718 print STDERR "Compression: ",
719 $compression_enabled? "enabled" : "disabled",
722 if ($compression_enabled and !$opt_force) {
723 print STDERR "Turn off compression or run amtapetype with the -f option\n";
727 if (!$opt_only_compression) {
728 make_tapetype($compression_enabled);
732 Amanda::Util::finish_application();