2 # Copyright (c) 2008 Zmanda Inc. All Rights Reserved.
4 # This program is free software; you can redistribute it and/or modify it
5 # under the terms of the GNU General Public License version 2 as published
6 # by the Free Software Foundation.
8 # This program is distributed in the hope that it will be useful, but
9 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 # Contact information: Zmanda Inc., 465 S Mathlida Ave, Suite 300
18 # Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
20 # This is a tool to examine a device and generate a reasonable tapetype
23 use lib '@amperldir@';
29 use Amanda::BigIntCompat;
31 use Amanda::Device qw( :constants );
32 use Amanda::Debug qw( :logging );
33 use Amanda::Util qw( :constants );
34 use Amanda::Config qw( :init :getconf config_dir_relative );
37 use Amanda::Constants;
40 # command-line options
41 my $opt_only_compression = 0;
43 my $opt_tapetype_name = 'unknown-tapetype';
45 my $opt_label = "amtapetype-".(int rand 2**31);
49 # global "hint" from the compression heuristic as to how fast this
51 my $device_speed_estimate;
53 # open up a device, optionally check its label, and start it in ACCESS_WRITE.
55 my $device = Amanda::Device->new($opt_device_name);
56 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
57 die("Could not open device $opt_device_name: ".$device->error()."\n");
60 if (defined $opt_blocksize) {
61 $device->property_set('BLOCK_SIZE', $opt_blocksize)
62 or die "Error setting blocksize: " . $device->error_or_status();
66 my $read_label_status = $device->read_label();
67 if ($read_label_status & $DEVICE_STATUS_VOLUME_UNLABELED) {
68 if ($device->volume_label) {
69 die "Volume in device $opt_device_name has Amanda label '" .
70 {$device->volume_label} . "'. Giving up.";
72 } elsif ($read_label_status != $DEVICE_STATUS_SUCCESS) {
73 die "Error reading label: " . $device->error_or_status();
83 if (!$device->start($ACCESS_WRITE, $opt_label, undef)) {
84 die("Error writing label '$opt_label': ". $device->error_or_status());
90 # Write a single file to the device, and record the results in STATS.
92 # STATS => $stats_hashref, (see below)
93 # DEVICE => $dev, (device to write to)
94 # PATTERN => RANDOM or FIXED, (data pattern to write)
95 # BYTES => nn, (number of bytes; optional)
96 # MAX_TIME => secs); (cancel write after this time; optional)
98 # Returns 0 on success (including EOM), "TIMEOUT" on timeout, or an error message
101 # STATS is a multi-level hashref; write_one_file adds to any values
102 # already in the data structure.
103 # $stats->{$pattern}->{TIME} - number of seconds spent writing
104 # $stats->{$pattern}->{FILES} - number of files written
105 # $stats->{$pattern}->{BYTES} - number of bytes written (approximate)
107 sub write_one_file(%) {
109 my $stats = $options{'STATS'} || { };
110 my $device = $options{'DEVICE'};
111 my $bytes = $options{'MAX_BYTES'} || 0;
112 my $pattern = $options{'PATTERN'} || 'FIXED';
113 my $max_time = $options{'MAX_TIME'} || 0;
116 my $hdr = Amanda::Types::dumpfile_t->new();
117 $hdr->{type} = $Amanda::Types::F_DUMPFILE;
118 $hdr->{name} = "amtapetype";
119 $hdr->{disk} = "/test";
120 $hdr->{datestamp} = "X";
121 $device->start_file($hdr)
122 or return $device->error_or_status();
124 # set up the transfer
125 my ($source, $dest, $xfer);
126 if ($pattern eq 'FIXED') {
127 # a simple 256-byte pattern to dodge run length encoding.
128 my $non_random_pattern = pack("C*", 0..255);
129 $source = Amanda::Xfer::Source::Pattern->new($bytes, $non_random_pattern);
130 } elsif ($pattern eq 'RANDOM') {
131 $source = Amanda::Xfer::Source::Random->new($bytes, 1 + int rand 100);
133 die "Unknown PATTERN $pattern";
135 $dest = Amanda::Xfer::Dest::Device->new($device, 0);
136 $xfer = Amanda::Xfer->new([$source, $dest]);
138 # set up the relevant callbacks
139 my ($timeout_src, $xfer_src, $spinner_src);
143 $xfer_src = $xfer->get_source();
144 $xfer_src->set_callback(sub {
145 my ($src, $xmsg, $xfer) = @_;
146 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
147 $got_error = $xmsg->{message};
149 if ($xfer->get_status() == $Amanda::Xfer::XFER_DONE) {
150 Amanda::MainLoop::quit();
155 $timeout_src = Amanda::MainLoop::timeout_source($max_time * 1000);
156 $timeout_src->set_callback(sub {
159 $xfer->cancel(); # will result in an XFER_DONE
163 $spinner_src = Amanda::MainLoop::timeout_source(1000);
164 $spinner_src->set_callback(sub {
166 my ($file, $block) = ($device->file(), $device->block());
167 print STDERR "File $file, block $block \r";
170 my $start_time = time();
173 Amanda::MainLoop::run();
175 $spinner_src->remove();
176 $timeout_src->remove() if ($timeout_src);
177 print STDERR " " x 60, "\r";
179 my $duration = time() - $start_time;
181 # OK, we finished, update statistics (even if we saw an error)
182 my $blocks_written = $device->block();
183 my $block_size = $device->property_get("block_size");
184 $stats->{$pattern}->{BYTES} += $blocks_written * $block_size;
185 $stats->{$pattern}->{FILES} += 1;
186 $stats->{$pattern}->{TIME} += $duration;
188 if ($device->status() != $Amanda::Device::DEVICE_STATUS_SUCCESS) {
189 return $device->error_or_status();
203 sub check_compression {
206 # Check compression status here by property query. If the device can answer
207 # the question, there's no reason to investigate further.
208 my $compression_enabled = $device->property_get("compression");
210 if (defined $compression_enabled) {
211 return $compression_enabled;
214 # Need to use heuristic to find out if compression is enabled. Also, we
215 # rewind between passes so that the second pass doesn't get some kind of
216 # buffering advantage.
218 print STDERR "Applying heuristic check for compression.\n";
220 # We base our determination on whether it's faster to write random data or
221 # patterned data. That starts by writing random data for a short length of
222 # time, then measuring the elapsed time and total data written. Due to
223 # potential delay in cancelling a transfer, the elapsed time will be a bit
224 # longer than the intended time. We then write the same amount of
225 # patterned data, and again measure the elapsed time. We can then
226 # calculate the speeds of the two operations. If the compressible speed
227 # was faster by more than min_ratio, then we assume compression is enabled.
229 my $compression_check_time = 60;
230 my $compression_check_min_ratio = 1.2;
234 start_device($device);
236 my $err = write_one_file(
239 MAX_TIME => $compression_check_time,
240 PATTERN => 'RANDOM');
242 if ($err != 'TIMEOUT') {
246 # restart the device to rewind it
247 start_device($device);
249 $err = write_one_file(
252 MAX_BYTES => $stats->{'RANDOM'}->{'BYTES'},
258 # speed calculations are a little tricky: BigInt * float comes out to NaN, so we
259 # cast the BigInts to float first
260 my $random_speed = ($stats->{RANDOM}->{BYTES} . "") / $stats->{RANDOM}->{TIME};
261 my $fixed_speed = ($stats->{FIXED}->{BYTES} . "") / $stats->{FIXED}->{TIME};
263 print STDERR "Wrote random (uncompressible) data at $random_speed bytes/sec\n";
264 print STDERR "Wrote fixed (compressible) data at $fixed_speed bytes/sec\n";
266 # sock this away for make_tapetype's use
267 $device_speed_estimate = $random_speed;
269 $compression_enabled =
270 ($fixed_speed / $random_speed > $compression_check_min_ratio);
271 return $compression_enabled;
278 my $xfer = Amanda::Xfer->new([
279 Amanda::Xfer::Source::Device->new($device),
280 Amanda::Xfer::Dest::Null->new(0),
283 $xfer->get_source()->set_callback(sub {
284 my ($src, $xmsg, $xfer) = @_;
285 if ($xmsg->{type} == $Amanda::Xfer::XMSG_ERROR) {
286 $got_error = $xmsg->{message};
288 if ($xfer->get_status() == $Amanda::Xfer::XFER_DONE) {
289 Amanda::MainLoop::quit();
294 Amanda::MainLoop::run();
300 my $fsf_after_filemark = $device->property_get("FSF_AFTER_FILEMARK");
302 # not a 'tape:' device
303 return if !defined $fsf_after_filemark;
305 $device->start($ACCESS_WRITE, "TEST-001", "20080706050403");
307 my $hdr = Amanda::Types::dumpfile_t->new();
309 $hdr->{type} = $Amanda::Types::F_DUMPFILE;
310 $hdr->{name} = "localhost";
311 $hdr->{disk} = "/test1";
312 $hdr->{datestamp} = "20080706050403";
313 $device->start_file($hdr);
314 $device->finish_file();
316 $hdr->{type} = $Amanda::Types::F_DUMPFILE;
317 $hdr->{name} = "localhost";
318 $hdr->{disk} = "/test2";
319 $hdr->{datestamp} = "20080706050403";
320 $device->start_file($hdr);
321 $device->finish_file();
323 $hdr->{type} = $Amanda::Types::F_DUMPFILE;
324 $hdr->{name} = "localhost";
325 $hdr->{disk} = "/test3";
326 $hdr->{datestamp} = "20080706050403";
327 $device->start_file($hdr);
328 $device->finish_file();
332 #set fsf_after_filemark to false
333 $device->property_set('FSF_AFTER_FILEMARK', 0)
334 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
336 my $need_fsf_after_filemark = 0;
338 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
339 die ("Could not read label from: " . $device->error_or_status());
341 if ($device->volume_label != "TEST-001") {
342 die ("wrong label: ", $device->volume_label);
344 $device->start($ACCESS_READ, undef, undef)
345 or die ("Could not start device: " . $device->error_or_status());
347 $hdr = $device->seek_file(1);
348 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
349 die ("seek_file(1) failed");
351 if ($hdr->{disk} ne "/test1") {
352 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
354 data_to_null($device);
356 $hdr = $device->seek_file(2);
357 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
358 if ($hdr->{disk} ne "/test2") {
359 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
361 data_to_null($device);
363 $hdr = $device->seek_file(3);
364 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
365 die("seek_file(3) failed");
367 if ($hdr->{disk} ne "/test3") {
368 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
370 data_to_null($device);
372 $need_fsf_after_filemark = 1;
377 #verify need_fsf_after_filemark
378 my $fsf_after_filemark_works = 0;
379 if ($need_fsf_after_filemark) {
380 #set fsf_after_filemark to true
381 $device->property_set('FSF_AFTER_FILEMARK', 1)
382 or die "Error setting FSF_AFTER_FILEMARK: " . $device->error_or_status();
384 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
385 die ("Could not read label from: " . $device->error_or_status());
387 if ($device->volume_label != "TEST-001") {
388 die ("wrong label: ", $device->volume_label);
390 $device->start($ACCESS_READ, undef, undef)
391 or die ("Could not start device: " . $device->error_or_status());
393 $hdr = $device->seek_file(1);
394 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
395 die ("seek_file(1) failed");
397 if ($hdr->{disk} ne "/test1") {
398 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
400 data_to_null($device);
402 $hdr = $device->seek_file(2);
403 if ($device->status() == $DEVICE_STATUS_SUCCESS) {
404 if ($hdr->{disk} ne "/test2") {
405 die ("Wrong disk: " . $hdr->{disk} . " expected /test2");
407 data_to_null($device);
409 $hdr = $device->seek_file(3);
410 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
411 die("seek_file(3) failed");
413 if ($hdr->{disk} ne "/test3") {
414 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
416 data_to_null($device);
417 $fsf_after_filemark_works = 1;
419 die("seek_file(2) failed");
424 if ($need_fsf_after_filemark == 0 && $fsf_after_filemark_works == 0) {
425 if (defined $opt_property || $fsf_after_filemark) {
426 print STDOUT "device_property \"FSF_AFTER_FILEMARK\" \"false\"\n";
428 $device->property_set('FSF_AFTER_FILEMARK', 0);
429 } elsif ($need_fsf_after_filemark == 1 && $fsf_after_filemark_works == 1) {
430 if (defined $opt_property || !$fsf_after_filemark) {
431 print STDOUT "device_property \"FSF_AFTER_FILEMARK\" \"true\"\n";
433 $device->property_set('FSF_AFTER_FILEMARK', 1);
435 die ("Broken seek_file");
438 #Check seek to file 1 from header
439 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
440 die ("Could not read label from: " . $device->error_or_status());
442 if ($device->volume_label != "TEST-001") {
443 die ("wrong label: ", $device->volume_label);
445 $device->start($ACCESS_READ, undef, undef)
446 or die ("Could not start device: " . $device->error_or_status());
448 $hdr = $device->seek_file(1);
449 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
450 die ("seek_file(1) failed");
452 if ($hdr->{disk} ne "/test1") {
453 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
457 #Check seek to file 2 from header
458 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
459 die ("Could not read label from: " . $device->error_or_status());
461 if ($device->volume_label != "TEST-001") {
462 die ("wrong label: ", $device->volume_label);
464 $device->start($ACCESS_READ, undef, undef)
465 or die ("Could not start device: " . $device->error_or_status());
467 $hdr = $device->seek_file(2);
468 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
469 die ("seek_file(2) failed");
471 if ($hdr->{disk} ne "/test2") {
472 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
476 #Check seek to file 3 from header
477 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
478 die ("Could not read label from: " . $device->error_or_status());
480 if ($device->volume_label != "TEST-001") {
481 die ("wrong label: ", $device->volume_label);
483 $device->start($ACCESS_READ, undef, undef)
484 or die ("Could not start device: " . $device->error_or_status());
486 $hdr = $device->seek_file(3);
487 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
488 die ("seek_file(3) failed");
490 if ($hdr->{disk} ne "/test3") {
491 die ("Wrong disk: " . $hdr->{disk} . " expected /test1");
495 #Check seek to file 3 from eof of 1
496 if ($device->read_label() != $DEVICE_STATUS_SUCCESS) {
497 die ("Could not read label from: " . $device->error_or_status());
499 if ($device->volume_label != "TEST-001") {
500 die ("wrong label: ", $device->volume_label);
502 $device->start($ACCESS_READ, undef, undef)
503 or die ("Could not start device: " . $device->error_or_status());
505 $hdr = $device->seek_file(1);
506 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
507 die ("seek_file(1) failed");
509 data_to_null($device);
510 $hdr = $device->seek_file(3);
511 if ($device->status() != $DEVICE_STATUS_SUCCESS) {
512 die ("seek_file(3) failed");
514 if ($hdr->{disk} ne "/test3") {
515 die ("Wrong disk: " . $hdr->{disk} . " expected /test3");
521 my ($device, $compression_enabled) = @_;
522 my $blocksize = $device->property_get("BLOCK_SIZE");
524 # First, write one very long file to get the total tape length
525 print STDERR "Writing one file to fill the volume.\n";
527 start_device($device);
528 my $err = write_one_file(
531 PATTERN => 'RANDOM');
533 if ($stats->{RANDOM}->{BYTES} < 1024 * 1024 * 100) {
534 die "Wrote less than 100MB to the device: $err\n";
536 my $volume_size_estimate = $stats->{RANDOM}->{BYTES};
537 my $speed_estimate = (($stats->{RANDOM}->{BYTES}."") / 1024)
538 / $stats->{RANDOM}->{TIME};
539 $speed_estimate = int $speed_estimate;
540 print STDERR "Wrote $volume_size_estimate bytes at $speed_estimate kb/sec\n";
542 # now we want to write about 100 filemarks; round down to the blocksize
543 # to avoid counting padding as part of the filemark
544 my $file_size = $volume_size_estimate / 100;
545 $file_size -= $file_size % $blocksize;
547 print STDERR "Writing smaller files ($file_size bytes) to determine filemark.\n";
549 start_device($device);
550 while (!write_one_file(
553 MAX_BYTES => $file_size,
554 PATTERN => 'RANDOM')) { }
556 my $filemark_estimate = ($volume_size_estimate - $stats->{RANDOM}->{BYTES})
557 / ($stats->{RANDOM}->{FILES} - 1);
558 if ($filemark_estimate < 0) {
559 $filemark_estimate = 0;
562 my $comment = "Created by amtapetype; compression "
563 . ($compression_enabled? "enabled" : "disabled");
565 # round these parameters to the nearest kb, since the parameters' units
567 my $volume_size_estimate_kb = $volume_size_estimate/1024;
568 my $filemark_kb = $filemark_estimate/1024;
570 # and suggest using device_property for blocksize if it's not an even multiple
573 if ($blocksize % 1024 == 0) {
574 $blocksize_line = "blocksize " . $blocksize/1024 . " kbytes";
576 $blocksize_line = "# add device_property \"BLOCK_SIZE\" \"$blocksize\" to the device";
580 define tapetype $opt_tapetype_name {
582 length $volume_size_estimate_kb kbytes
583 filemark $filemark_kb kbytes
584 speed $speed_estimate kps
592 Usage: amtapetype [-h] [-c] [-f] [-b blocksize] [-t typename] [-l label]
593 [ [-o config_overwrite] ... ] device
594 -h Display this message
595 -c Only check hardware compression state
596 -f Run amtapetype even if the loaded volume is already in use
597 or compression is enabled.
598 -b Blocksize to use (default 32k)
599 -t Name to give to the new tapetype definition
600 -l Label to write to the tape (default is randomly generated)
601 -p Check property of the device.
602 -o Overwrite configuration parameter (such as device properties)
603 Blocksize can include an optional suffix (k, m, or g)
608 ## Application initialization
610 Amanda::Util::setup_application("amtapetype", "server", $CONTEXT_CMDLINE);
611 config_init(0, undef);
613 my $config_overwrites = new_config_overwrites($#ARGV+1);
615 Getopt::Long::Configure(qw(bundling));
617 'help|usage|?|h' => \&usage,
618 'c' => \$opt_only_compression,
620 my ($num, $suff) = ($_[1] =~ /^([0-9]+)\s*(.*)$/);
621 die "Invalid blocksize '$_[1]'" unless (defined $num);
622 my $mult = (defined $suff)?
623 Amanda::Config::find_multiplier($suff) : 1;
624 die "Invalid suffix '$suff'" unless ($mult);
625 $opt_blocksize = $num * $mult;
627 't=s' => \$opt_tapetype_name,
630 'p' => \$opt_property,
631 'o=s' => sub { add_config_overwrite_opt($config_overwrites, $_[1]); },
633 usage() if (@ARGV != 1);
635 $opt_device_name= shift @ARGV;
637 apply_config_overwrites($config_overwrites);
638 my ($cfgerr_level, @cfgerr_errors) = config_errors();
639 if ($cfgerr_level >= $CFGERR_WARNINGS) {
640 config_print_errors();
641 if ($cfgerr_level >= $CFGERR_ERRORS) {
642 die("errors processing configuration options");
646 Amanda::Util::finish_setup($RUNNING_AS_ANY);
648 my $device = open_device();
650 # Find property of the device.
651 check_property($device);
653 if (!defined $opt_property) {
654 my $compression_enabled = check_compression($device);
655 print STDERR "Compression: ",
656 $compression_enabled? "enabled" : "disabled",
659 if ($compression_enabled and !$opt_force) {
660 print STDERR "Turn off compression or run amtapetype with the -f option\n";
664 if (!$opt_only_compression) {
665 make_tapetype($device, $compression_enabled);