Merge branch 'master' into squeeze
[debian/amanda] / perl / Amanda / Changer / rait.pm
diff --git a/perl/Amanda/Changer/rait.pm b/perl/Amanda/Changer/rait.pm
new file mode 100644 (file)
index 0000000..26486bc
--- /dev/null
@@ -0,0 +1,724 @@
+# Copyright (c) 2009,2010 Zmanda, Inc.  All Rights Reserved.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 as published
+# by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+#
+# Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
+# Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
+
+package Amanda::Changer::rait;
+
+use strict;
+use warnings;
+use vars qw( @ISA );
+@ISA = qw( Amanda::Changer );
+
+use File::Glob qw( :glob );
+use File::Path;
+use Amanda::Config qw( :getconf );
+use Amanda::Debug qw( debug warning );
+use Amanda::Util qw( :alternates );
+use Amanda::Changer;
+use Amanda::MainLoop;
+use Amanda::Device qw( :constants );
+
+=head1 NAME
+
+Amanda::Changer::rait
+
+=head1 DESCRIPTION
+
+This changer operates several child changers, returning RAIT devices composed of
+the devices produced by the child changers.  It's modeled on the RAIT device.
+
+See the amanda-changers(7) manpage for usage information.
+
+=cut
+
+sub new {
+    my $class = shift;
+    my ($config, $tpchanger) = @_;
+    my ($kidspecs) = ( $tpchanger =~ /chg-rait:(.*)/ );
+
+    my @kidspecs = Amanda::Util::expand_braced_alternates($kidspecs);
+    if (@kidspecs < 2) {
+       return Amanda::Changer->make_error("fatal", undef,
+           message => "chg-rait needs at least two child changers");
+    }
+
+    my @children = map {
+       ($_ eq "ERROR")? "ERROR" : Amanda::Changer->new($_)
+    } @kidspecs;
+
+    if (grep { $_->isa("Amanda::Changer::Error") } @children) {
+       my @annotated_errs;
+       for my $i (0 .. @children-1) {
+           next unless $children[$i]->isa("Amanda::Changer::Error");
+           if ($children[$i]->isa("Amanda::Changer::Error")) {
+               push @annotated_errs,
+                   [ $kidspecs[$i], $children[$i] ];
+           } elsif ($children[$i]->isa("Amanda::Changer")) {
+               $children[$i]->quit();
+           }
+       }
+       return Amanda::Changer->make_combined_error(
+               "fatal", [ @annotated_errs ]);
+    }
+
+    my $self = {
+       config => $config,
+       child_names => \@kidspecs,
+       children => \@children,
+       num_children => scalar @children,
+    };
+    bless ($self, $class);
+    return $self;
+}
+
+sub quit {
+    my $self = shift;
+
+    # quit each child
+    foreach my $child (@{$self->{'children'}}) {
+       $child->quit() if $child ne "ERROR";
+    }
+
+    $self->SUPER::quit();
+}
+
+# private method to help handle slot input
+sub _kid_slots_ok {
+    my ($self, $res_cb, $slot, $kid_slots_ref, $err_ref) = @_;
+    @{$kid_slots_ref} = expand_braced_alternates($slot);
+    return 1 if (@{$kid_slots_ref} == $self->{'num_children'});
+
+    if (@{$kid_slots_ref} == 1) {
+        @{$kid_slots_ref} = ( $slot ) x $self->{'num_children'};
+        return 1;
+    }
+    ${$err_ref} = $self->make_error("failed", $res_cb,
+                                    reason => "invalid",
+                                    message => "slot string '$slot' does not specify " .
+                                    "$self->{num_children} child slots");
+    return 0;
+}
+
+sub load {
+    my $self = shift;
+    my %params = @_;
+
+    return if $self->check_error($params{'res_cb'});
+
+    $self->validate_params('load', \%params);
+
+    my $release_on_error = sub {
+       my ($kid_results) = @_;
+
+       # an error has occurred, so we have to release all of the *non*-error
+       # reservations (and handle errors in those releases!), then construct
+       # and return a combined error message.
+
+       my $releases_outstanding = 1; # start at one, in case the releases are immediate
+       my @release_errors = ( undef ) x $self->{'num_children'};
+       my $releases_maybe_done = sub {
+           return if (--$releases_outstanding);
+
+           # gather up the errors and combine them for return to our caller
+           my @annotated_errs;
+           for my $i (0 .. $self->{'num_children'}-1) {
+               my $child_name = $self->{'child_names'}[$i];
+               if ($kid_results->[$i][0]) {
+                   push @annotated_errs,
+                       [ "from $child_name", $kid_results->[$i][0] ];
+               }
+               if ($release_errors[$i]) {
+                   push @annotated_errs,
+                       [ "while releasing $child_name reservation",
+                         $kid_results->[$i][0] ];
+               }
+           }
+
+           return $self->make_combined_error(
+               $params{'res_cb'}, [ @annotated_errs ]);
+       };
+
+       for my $i (0 .. $self->{'num_children'}-1) {
+           next unless (my $res = $kid_results->[$i][1]);
+           $releases_outstanding++;
+           $res->release(finished_cb => sub {
+               $release_errors[$i] = $_[0];
+               $releases_maybe_done->();
+           });
+       }
+
+       # we started $releases_outstanding at 1, so decrement it now
+       $releases_maybe_done->();
+    };
+
+    my $all_kids_done_cb = sub {
+       my ($kid_results) = @_;
+       my $result;
+
+       # first, let's see if any changer gave an error
+       if (!grep { defined($_->[0]) } @$kid_results) {
+           # no error .. combine the reservations and return a RAIT reservation
+           return $self->_make_res($params{'res_cb'}, [ map { $_->[1] } @$kid_results ]);
+       } else {
+           return $release_on_error->($kid_results);
+       }
+    };
+
+    # make a template for params for the children
+    my %kid_template = %params;
+    delete $kid_template{'res_cb'};
+    delete $kid_template{'slot'};
+    delete $kid_template{'except_slots'};
+    # $kid_template{'label'} is passed directly to children
+    # $kid_template{'relative_slot'} is passed directly to children
+    # $kid_template{'mode'} is passed directly to children
+
+    # and make a copy for each child
+    my @kid_params;
+    for (0 .. $self->{'num_children'}-1) {
+       push @kid_params, { %kid_template };
+    }
+
+    if (exists $params{'slot'}) {
+       my $slot = $params{'slot'};
+
+       # calculate the slots for each child
+       my (@kid_slots, $err);
+        return $err unless $self->_kid_slots_ok($params{'res_cb'}, $slot, \@kid_slots, \$err);
+       if (@kid_slots != $self->{'num_children'}) {
+           # as a convenience, expand a single slot into the same slot for each child
+           if (@kid_slots == 1) {
+               @kid_slots = ( $slot ) x $self->{'num_children'};
+           } else {
+               return $self->make_error("failed", $params{'res_cb'},
+                       reason => "invalid",
+                       message => "slot '$slot' does not specify " .
+                                   "$self->{num_children} child slots");
+           }
+       }
+       for (0 .. $self->{'num_children'}-1) {
+           $kid_params[$_]->{'slot'} = $kid_slots[$_];
+       }
+    }
+
+    # each slot in except_slots needs to get broken down, and the appropriate slot
+    # given to each child
+    if (exists $params{'except_slots'}) {
+       for (0 .. $self->{'num_children'}-1) {
+           $kid_params[$_]->{'except_slots'} = {};
+       }
+
+       # for each slot, split it up, then apportion the result to each child
+       for my $slot ( keys %{$params{'except_slots'}} ) {
+           my (@kid_slots, $err);
+            return $err unless $self->_kid_slots_ok($params{'res_cb'}, $slot, \@kid_slots, \$err);
+           for (0 .. $self->{'num_children'}-1) {
+               $kid_params[$_]->{'except_slots'}->{$kid_slots[$_]} = 1;
+           }
+       }
+    }
+
+    $self->_for_each_child(
+       oksub => sub {
+           my ($kid_chg, $kid_cb, $kid_params) = @_;
+           $kid_params->{'res_cb'} = $kid_cb;
+           $kid_chg->load(%$kid_params);
+       },
+       errsub => sub {
+           my ($kid_chg, $kid_cb, $kid_slot) = @_;
+           $kid_cb->(undef, "ERROR");
+       },
+       parent_cb => $all_kids_done_cb,
+       args => \@kid_params,
+    );
+}
+
+sub _make_res {
+    my $self = shift;
+    my ($res_cb, $kid_reservations) = @_;
+    my @kid_devices = map { ($_ ne "ERROR") ? $_->{'device'} : undef } @$kid_reservations;
+
+    my $rait_device = Amanda::Device->new_rait_from_children(@kid_devices);
+    if ($rait_device->status() != $DEVICE_STATUS_SUCCESS) {
+       return $self->make_error("failed", $res_cb,
+               reason => "device",
+               message => $rait_device->error_or_status());
+    }
+
+    if (my $err = $self->{'config'}->configure_device($rait_device)) {
+       return $self->make_error("failed", $res_cb,
+               reason => "device",
+               message => $err);
+    }
+
+    my $combined_res = Amanda::Changer::rait::Reservation->new(
+       $kid_reservations, $rait_device);
+    $rait_device->read_label();
+
+    $res_cb->(undef, $combined_res);
+}
+
+sub info_key {
+    my $self = shift;
+    my ($key, %params) = @_;
+
+    return if $self->check_error($params{'info_cb'});
+
+    my $check_and_report_errors = sub {
+       my ($kid_results) = @_;
+
+       if (grep { defined($_->[0]) } @$kid_results) {
+           # we have errors, so collect them and make a "combined" error.
+           my @annotated_errs;
+           my @err_slots;
+           for my $i (0 .. $self->{'num_children'}-1) {
+               my $kr = $kid_results->[$i];
+               next unless defined($kr->[0]);
+               push @annotated_errs,
+                   [ $self->{'child_names'}[$i], $kr->[0] ];
+               push @err_slots, $kr->[0]->{'slot'}
+                   if (defined $kr->[0] and defined $kr->[0]->{'slot'});
+           }
+
+           my @slotarg;
+           if (@err_slots == $self->{'num_children'}) {
+               @slotarg = (slot => collapse_braced_alternates([@err_slots]));
+           }
+
+           $self->make_combined_error(
+               $params{'info_cb'}, [ @annotated_errs ],
+               @slotarg);
+           return 1;
+       }
+    };
+
+    if ($key eq 'num_slots') {
+       my $all_kids_done_cb = sub {
+           my ($kid_results) = @_;
+           return if ($check_and_report_errors->($kid_results));
+
+           # aggregate the results: the consensus if the children agree,
+           # otherwise -1
+           my $num_slots;
+           for (@$kid_results) {
+               my ($err, %kid_info) = @$_;
+               next unless exists($kid_info{'num_slots'});
+               my $kid_num_slots = $kid_info{'num_slots'};
+               if (defined $num_slots and $num_slots != $kid_num_slots) {
+                   debug("chg-rait: children have different slot counts!");
+                   $num_slots = -1;
+               } else {
+                   $num_slots = $kid_num_slots;
+               }
+           }
+           $params{'info_cb'}->(undef, num_slots => $num_slots) if $params{'info_cb'};
+       };
+
+       $self->_for_each_child(
+           oksub => sub {
+               my ($kid_chg, $kid_cb) = @_;
+               $kid_chg->info(info => [ 'num_slots' ], info_cb => $kid_cb);
+           },
+           errsub => undef,
+           parent_cb => $all_kids_done_cb,
+       );
+    } elsif ($key eq "vendor_string") {
+       my $all_kids_done_cb = sub {
+           my ($kid_results) = @_;
+           return if ($check_and_report_errors->($kid_results));
+
+           my @kid_vendors =
+               grep { defined($_) }
+               map { my ($e, %r) = @$_; $r{'vendor_string'} }
+               @$kid_results;
+           my $vendor_string;
+           if (@kid_vendors) {
+               $vendor_string = collapse_braced_alternates([@kid_vendors]);
+               $params{'info_cb'}->(undef, vendor_string => $vendor_string) if $params{'info_cb'};
+           } else {
+               $params{'info_cb'}->(undef) if $params{'info_cb'};
+           }
+       };
+
+       $self->_for_each_child(
+           oksub => sub {
+               my ($kid_chg, $kid_cb) = @_;
+               $kid_chg->info(info => [ 'vendor_string' ], info_cb => $kid_cb);
+           },
+           errsub => undef,
+           parent_cb => $all_kids_done_cb,
+       );
+    } elsif ($key eq 'fast_search') {
+       my $all_kids_done_cb = sub {
+           my ($kid_results) = @_;
+           return if ($check_and_report_errors->($kid_results));
+
+           my @kid_fastness =
+               grep { defined($_) }
+               map { my ($e, %r) = @$_; $r{'fast_search'} }
+               @$kid_results;
+           if (@kid_fastness) {
+               my $fast_search = 1;
+               # conduct a logical AND of all child fastnesses
+               for my $f (@kid_fastness) {
+                   $fast_search = $fast_search && $f;
+               }
+               $params{'info_cb'}->(undef, fast_search => $fast_search) if $params{'info_cb'};
+           } else {
+               $params{'info_cb'}->(undef, fast_search => 0) if $params{'info_cb'};
+           }
+       };
+
+       $self->_for_each_child(
+           oksub => sub {
+               my ($kid_chg, $kid_cb) = @_;
+               $kid_chg->info(info => [ 'fast_search' ], info_cb => $kid_cb);
+           },
+           errsub => undef,
+           parent_cb => $all_kids_done_cb,
+       );
+    }
+}
+
+# reset, clean, etc. are all *very* similar to one another, so we create them
+# generically
+sub _mk_simple_op {
+    my ($op, $has_drive) = @_;
+    sub {
+       my $self = shift;
+       my %params = @_;
+
+       return if $self->check_error($params{'finished_cb'});
+
+       my $all_kids_done_cb = sub {
+           my ($kid_results) = @_;
+           if (grep { defined($_->[0]) } @$kid_results) {
+               # we have errors, so collect them and make a "combined" error.
+               my @annotated_errs;
+               for my $i (0 .. $self->{'num_children'}-1) {
+                   my $kr = $kid_results->[$i];
+                   next unless defined($kr->[0]);
+                   push @annotated_errs,
+                       [ $self->{'child_names'}[$i], $kr->[0] ];
+               }
+               $self->make_combined_error(
+                   $params{'finished_cb'}, [ @annotated_errs ]);
+               return 1;
+           }
+           $params{'finished_cb'}->() if $params{'finished_cb'};
+       };
+
+       # get the drives for the kids, if necessary
+       my @kid_args;
+       if ($has_drive and $params{'drive'}) {
+           my $drive = $params{'drive'};
+           my @kid_drives = expand_braced_alternates($drive);
+
+           if (@kid_drives == 1) {
+               @kid_drives = ( $kid_drives[0] ) x $self->{'num_children'};
+           }
+
+           if (@kid_drives != $self->{'num_children'}) {
+               return $self->make_error("failed", $params{'finished_cb'},
+                       reason => "invalid",
+                       message => "drive string '$drive' does not specify " .
+                       "$self->{num_children} child drives");
+           }
+
+           @kid_args = map { { drive => $_ } } @kid_drives;
+           delete $params{'drive'};
+       } else {
+           @kid_args = ( {} ) x $self->{'num_children'};
+       }
+
+       $self->_for_each_child(
+           oksub => sub {
+               my ($kid_chg, $kid_cb, $args) = @_;
+               $kid_chg->$op(%params, finished_cb => $kid_cb, %$args);
+           },
+           errsub => undef,
+           parent_cb => $all_kids_done_cb,
+           args => \@kid_args,
+       );
+    };
+}
+
+{
+    # perl doesn't like that these symbols are only mentioned once
+    no warnings;
+
+    *reset = _mk_simple_op("reset", 0);
+    *update = _mk_simple_op("update", 0);
+    *clean = _mk_simple_op("clean", 1);
+    *eject = _mk_simple_op("eject", 1);
+}
+
+sub inventory {
+    my $self = shift;
+    my %params = @_;
+
+    return if $self->check_error($params{'inventory_cb'});
+
+    my $all_kids_done_cb = sub {
+       my ($kid_results) = @_;
+       if (grep { defined($_->[0]) } @$kid_results) {
+           # we have errors, so collect them and make a "combined" error.
+           my @annotated_errs;
+           for my $i (0 .. $self->{'num_children'}-1) {
+               my $kr = $kid_results->[$i];
+               next unless defined($kr->[0]);
+               push @annotated_errs,
+                   [ $self->{'child_names'}[$i], $kr->[0] ];
+           }
+           return $self->make_combined_error(
+               $params{'inventory_cb'}, [ @annotated_errs ]);
+       }
+
+       my $inv = $self->_merge_inventories($kid_results);
+       if (!defined $inv) {
+           return $self->make_error("failed", $params{'inventory_cb'},
+                   reason => "notimpl",
+                   message => "could not generate consistent inventory from rait child changers");
+       }
+
+       $params{'inventory_cb'}->(undef, $inv);
+    };
+
+    $self->_for_each_child(
+       oksub => sub {
+           my ($kid_chg, $kid_cb) = @_;
+           $kid_chg->inventory(inventory_cb => $kid_cb);
+       },
+       errsub => undef,
+       parent_cb => $all_kids_done_cb,
+    );
+}
+
+# Takes keyword parameters 'oksub', 'errsub', 'parent_cb', and 'args'.  For
+# each child, runs $oksub (or, if the child is "ERROR", $errsub), passing it
+# the changer, an aggregating callback, and the corresponding element from
+# @$args (if specified).  The callback combines its results with the results
+# from other changers, and when all results are available, calls $parent_cb.
+#
+# This forms a kind of "AND" combinator for a parallel operation on multiple
+# changers, providing the caller with a simple collection of the results of
+# the operation. The parent_cb is called as
+#   $parent_cb->([ [ <chg_1_results> ], [ <chg_2_results> ], .. ]).
+sub _for_each_child {
+    my $self = shift;
+    my %params = @_;
+    my ($oksub, $errsub, $parent_cb, $args) =
+       ($params{'oksub'}, $params{'errsub'}, $params{'parent_cb'}, $params{'args'});
+
+    if (defined($args)) {
+       die "number of args did not match number of children"
+           unless (@$args == $self->{'num_children'});
+    } else {
+       $args = [ ( undef ) x $self->{'num_children'} ];
+    }
+
+    my $remaining = $self->{'num_children'};
+    my @results = ( undef ) x $self->{'num_children'};
+    my $maybe_done = sub {
+       return if (--$remaining);
+       $parent_cb->([ @results ]);
+    };
+
+    for my $i (0 .. $self->{'num_children'}-1) {
+       my $child = $self->{'children'}[$i];
+       my $arg = @$args? $args->[$i] : undef;
+
+       my $child_cb = sub {
+           $results[$i] = [ @_ ];
+           $maybe_done->();
+       };
+
+       if ($child eq "ERROR") {
+           if (defined $errsub) {
+               $errsub->("ERROR", $child_cb, $arg);
+           } else {
+               # no errsub; just call $child_cb directly
+               $child_cb->(undef) if $child_cb;
+           }
+       } else {
+           $oksub->($child, $child_cb, $arg) if $oksub;
+       }
+    }
+}
+
+sub _merge_inventories {
+    my $self = shift;
+    my ($kid_results) = @_;
+
+    my @combined;
+    for my $kid_result (@$kid_results) {
+       my $kid_inv = $kid_result->[1];
+
+       if (!@combined) {
+           for my $x (@$kid_inv) {
+               push @combined, {
+                   state => Amanda::Changer::SLOT_FULL,
+                   device_status => undef, f_type => undef,
+                   label => undef, barcode => [],
+                   reserved => 0, slot => [],
+                   import_export => 1, loaded_in => [],
+               };
+           }
+       }
+
+       # if the results have different lengths, then we'll just call it
+       # not implemented; otherwise, we assume that the order of the slots
+       # in each child changer is the same.
+       if (scalar @combined != scalar @$kid_inv) {
+           warning("child changers returned different-length inventories; cannot merge");
+           return undef;
+       }
+
+       my $i;
+       for ($i = 0; $i < @combined; $i++) {
+           my $c = $combined[$i];
+           my $k = $kid_inv->[$i];
+           # mismatches here are just warnings
+           if (defined $c->{'label'}) {
+               if (defined $k->{'label'} and $c->{'label'} ne $k->{'label'}) {
+                   warning("child changers have different labels in slot at index $i");
+                   $c->{'label_mismatch'} = 1;
+                   $c->{'label'} = undef;
+               } elsif (!defined $k->{'label'}) {
+                   $c->{'label_mismatch'} = 1;
+                   $c->{'label'} = undef;
+               }
+           } else {
+               if (!$c->{'label_mismatch'} && !$c->{'label_set'}) {
+                   $c->{'label'} = $k->{'label'};
+               }
+           }
+           $c->{'label_set'} = 1;
+
+           $c->{'device_status'} |= $k->{'device_status'}
+               if defined $k->{'device_status'};
+
+           if (!defined $c->{'f_type'} ||
+               $k->{'f_type'} != $Amanda::Header::F_TAPESTART) {
+               $c->{'f_type'} = $k->{'f_type'};
+           }
+           # a slot is empty if any of the child slots are empty
+           $c->{'state'} = Amanda::Changer::SLOT_EMPTY
+                       if $k->{'state'} == Amanda::Changer::SLOT_EMPTY;
+
+           # a slot is reserved if any of the child slots are reserved
+           $c->{'reserved'} = $c->{'reserved'} || $k->{'reserved'};
+
+           # a slot is import-export if all of the child slots are import_export
+           $c->{'import_export'} = $c->{'import_export'} && $k->{'import_export'};
+
+           # barcodes, slots, and loaded_in are lists
+           push @{$c->{'slot'}}, $k->{'slot'};
+           push @{$c->{'barcode'}}, $k->{'barcode'};
+           push @{$c->{'loaded_in'}}, $k->{'loaded_in'};
+       }
+    }
+
+    # now post-process the slots, barcodes, and loaded_in into braced-alternates notation
+    my $i;
+    for ($i = 0; $i < @combined; $i++) {
+       my $c = $combined[$i];
+
+       delete $c->{'label_mismatch'} if $c->{'label_mismatch'};
+       delete $c->{'label_set'} if $c->{'label_set'};
+
+       $c->{'slot'} = collapse_braced_alternates([ @{$c->{'slot'}} ]);
+
+       if (grep { !defined $_ } @{$c->{'barcode'}}) {
+           delete $c->{'barcode'};
+       } else {
+           $c->{'barcode'} = collapse_braced_alternates([ @{$c->{'barcode'}} ]);
+       }
+
+       if (grep { !defined $_ } @{$c->{'loaded_in'}}) {
+           delete $c->{'loaded_in'};
+       } else {
+           $c->{'loaded_in'} = collapse_braced_alternates([ @{$c->{'loaded_in'}} ]);
+       }
+    }
+
+    return [ @combined ];
+}
+
+package Amanda::Changer::rait::Reservation;
+
+use Amanda::Util qw( :alternates );
+use vars qw( @ISA );
+@ISA = qw( Amanda::Changer::Reservation );
+
+# utility function to act like 'map', but pass "ERROR" straight through
+# (this has to appear before it is used, because it has a prototype)
+sub errmap (&@) {
+    my $sub = shift;
+    return map { ($_ ne "ERROR")? $sub->($_) : "ERROR" } @_;
+}
+
+sub new {
+    my $class = shift;
+    my ($child_reservations, $rait_device) = @_;
+    my $self = Amanda::Changer::Reservation::new($class);
+
+    # note that $child_reservations may contain "ERROR" in place of a reservation
+
+    $self->{'child_reservations'} = $child_reservations;
+
+    $self->{'device'} = $rait_device;
+
+    my @slot_names;
+    @slot_names = errmap { "" . $_->{'this_slot'} } @$child_reservations;
+    $self->{'this_slot'} = collapse_braced_alternates(\@slot_names);
+
+    return $self;
+}
+
+sub do_release {
+    my $self = shift;
+    my %params = @_;
+    my $remaining = @{$self->{'child_reservations'}};
+    my @outer_errors;
+
+    my $maybe_finished = sub {
+       my ($err) = @_;
+       push @outer_errors, $err if ($err);
+       return if (--$remaining);
+
+       my $errstr;
+       if (@outer_errors) {
+           $errstr = join("; ", @outer_errors);
+       }
+
+       # unref the device, for good measure
+       $self->{'device'} = undef;
+
+       $params{'finished_cb'}->($errstr) if $params{'finished_cb'};
+    };
+
+    for my $res (@{$self->{'child_reservations'}}) {
+       # short-circuit an "ERROR" reservation
+       if ($res eq "ERROR") {
+           $maybe_finished->(undef);
+           next;
+       }
+       $res->release(%params, finished_cb => $maybe_finished);
+    }
+}
+