git.gag.com Git - debian/amanda/blob - device-src/s3-device.c

   1 /*
   2  * Copyright (c) 2005 Zmanda, Inc.  All Rights Reserved.
   3  *
   4  * This library is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU Lesser General Public License version 2.1 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This library is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  10  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  11  * License for more details.
  12  *
  13  * You should have received a copy of the GNU Lesser General Public License
  14  * along with this library; if not, write to the Free Software Foundation,
  15  * Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA.
  16  *
  17  * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
  18  * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
  19  */
  20
  21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
  22  * data.  It stores data in keys named with a user-specified prefix, inside a
  23  * user-specified bucket.  Data is stored in the form of numbered (large)
  24  * blocks.
  25  */
  26
  27 #include <string.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <dirent.h>
  32 #include <regex.h>
  33 #include <time.h>
  34 #include "util.h"
  35 #include "amanda.h"
  36 #include "conffile.h"
  37 #include "device.h"
  38 #include "s3-device.h"
  39 #include <curl/curl.h>
  40 #ifdef HAVE_OPENSSL_HMAC_H
  41 # include <openssl/hmac.h>
  42 #else
  43 # ifdef HAVE_CRYPTO_HMAC_H
  44 #  include <crypto/hmac.h>
  45 # else
  46 #  ifdef HAVE_HMAC_H
  47 #   include <hmac.h>
  48 #  endif
  49 # endif
  50 #endif
  51
  52 /*
  53  * Constants and static data
  54  */
  55
  56 /* Maximum key length as specified in the S3 documentation
  57  * (*excluding* null terminator) */
  58 #define S3_MAX_KEY_LENGTH 1024
  59
  60 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
  61 #define S3_DEVICE_MAX_BLOCK_SIZE (10*1024*1024)
  62
  63 /* This goes in lieu of file number for metadata. */
  64 #define SPECIAL_INFIX "special-"
  65
  66 /* pointer to the class of our parent */
  67 static DeviceClass *parent_class = NULL;
  68
  69 /*
  70  * prototypes
  71  */
  72
  73 /*
  74  * utility functions */
  75
  76 /* Given file and block numbers, return an S3 key.
  77  *
  78  * @param self: the S3Device object
  79  * @param file: the file number
  80  * @param block: the block within that file
  81  * @returns: a newly allocated string containing an S3 key.
  82  */
  83 static char *
  84 file_and_block_to_key(S3Device *self,
  85                       int file,
  86                       guint64 block);
  87
  88 /* Given the name of a special file (such as 'tapestart'), generate
  89  * the S3 key to use for that file.
  90  *
  91  * @param self: the S3Device object
  92  * @param special_name: name of the special file
  93  * @param file: a file number to include; omitted if -1
  94  * @returns: a newly alocated string containing an S3 key.
  95  */
  96 static char *
  97 special_file_to_key(S3Device *self,
  98                     char *special_name,
  99                     int file);
 100 /* Write an amanda header file to S3.
 101  *
 102  * @param self: the S3Device object
 103  * @param label: the volume label
 104  * @param timestamp: the volume timestamp
 105  */
 106 static gboolean
 107 write_amanda_header(S3Device *self,
 108                     char *label,
 109                     char * timestamp);
 110
 111 /* "Fast forward" this device to the end by looking up the largest file number
 112  * present and setting the current file number one greater.
 113  *
 114  * @param self: the S3Device object
 115  */
 116 static gboolean
 117 seek_to_end(S3Device *self);
 118
 119 /* Find the number of the last file that contains any data (even just a header).
 120  *
 121  * @param self: the S3Device object
 122  * @returns: the last file, or -1 in event of an error
 123  */
 124 static int
 125 find_last_file(S3Device *self);
 126
 127 /* Delete all blocks in the given file, including the filestart block
 128  *
 129  * @param self: the S3Device object
 130  * @param file: the file to delete
 131  */
 132 static gboolean
 133 delete_file(S3Device *self,
 134             int file);
 135
 136 /* Set up self->s3 as best as possible.  Unless SILENT is TRUE,
 137  * any problems will generate warnings (with g_warning).  Regardless,
 138  * the return value is TRUE iff self->s3 is useable.
 139  *
 140  * @param self: the S3Device object
 141  * @param silent: silence warnings
 142  * @returns: TRUE if the handle is set up
 143  */
 144 static gboolean
 145 setup_handle(S3Device * self,
 146              gboolean ignore_problems);
 147
 148 /*
 149  * class mechanics */
 150
 151 static void
 152 s3_device_init(S3Device * o);
 153
 154 static void
 155 s3_device_class_init(S3DeviceClass * c);
 156
 157 static void
 158 s3_device_finalize(GObject * o);
 159
 160 static Device*
 161 s3_device_factory(char * device_type,
 162                   char * device_name);
 163
 164 /*
 165  * virtual functions */
 166
 167 static gboolean
 168 s3_device_open_device(Device *pself,
 169                       char *device_name);
 170
 171 static ReadLabelStatusFlags s3_device_read_label(Device * self);
 172
 173 static gboolean
 174 s3_device_start(Device * self,
 175                 DeviceAccessMode mode,
 176                 char * label,
 177                 char * timestamp);
 178
 179 static gboolean
 180 s3_device_start_file(Device * self,
 181                      const dumpfile_t * jobInfo);
 182
 183 static gboolean
 184 s3_device_write_block(Device * self,
 185                       guint size,
 186                       gpointer data,
 187                       gboolean last);
 188
 189 static gboolean
 190 s3_device_finish_file(Device * self);
 191
 192 static dumpfile_t*
 193 s3_device_seek_file(Device *pself,
 194                     guint file);
 195
 196 static gboolean
 197 s3_device_seek_block(Device *pself,
 198                      guint64 block);
 199
 200 static int
 201 s3_device_read_block(Device * pself,
 202                      gpointer data,
 203                      int *size_req);
 204
 205 static gboolean
 206 s3_device_recycle_file(Device *pself,
 207                        guint file);
 208
 209 static gboolean s3_device_property_set(Device * p_self, DevicePropertyId id,
 210                                        GValue * val);
 211 static gboolean s3_device_property_get(Device * p_self, DevicePropertyId id,
 212                                        GValue * val);
 213 /*
 214  * Private functions
 215  */
 216
 217 /* {{{ file_and_block_to_key */
 218 static char *
 219 file_and_block_to_key(S3Device *self,
 220                       int file,
 221                       guint64 block)
 222 {
 223     char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
 224                                    self->prefix, file, (long long unsigned int)block);
 225     g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
 226     return s3_key;
 227 }
 228 /* }}} */
 229
 230 /* {{{ special_file_to_key */
 231 static char *
 232 special_file_to_key(S3Device *self,
 233                     char *special_name,
 234                     int file)
 235 {
 236     if (file == -1)
 237         return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
 238     else
 239         return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
 240 }
 241 /* }}} */
 242
 243 /* {{{ write_amanda_header */
 244 static gboolean
 245 write_amanda_header(S3Device *self,
 246                     char *label,
 247                     char * timestamp)
 248 {
 249     char * amanda_header = NULL;
 250     char * key = NULL;
 251     int header_size;
 252     gboolean header_fits, result;
 253     dumpfile_t * dumpinfo = NULL;
 254
 255     /* build the header */
 256     dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
 257     amanda_header = device_build_amanda_header(DEVICE(self), dumpinfo,
 258                                                &header_size, &header_fits);
 259     if (!header_fits) {
 260         fprintf(stderr,
 261                 _("Amanda tapestart header won't fit in a single block!\n"));
 262         g_free(amanda_header);
 263         return FALSE;
 264     }
 265
 266     /* write out the header and flush the uploads. */
 267     key = special_file_to_key(self, "tapestart", -1);
 268     result = s3_upload(self->s3, self->bucket, key, amanda_header, header_size);
 269     g_free(amanda_header);
 270     g_free(key);
 271
 272     if (!result) {
 273         fprintf(stderr, _("While writing amanda header: %s\n"),
 274                 s3_strerror(self->s3));
 275     }
 276     return result;
 277 }
 278 /* }}} */
 279
 280 /* {{{ seek_to_end */
 281 static gboolean
 282 seek_to_end(S3Device *self) {
 283     int last_file;
 284
 285     Device *pself = DEVICE(self);
 286
 287     last_file = find_last_file(self);
 288     if (last_file < 0)
 289         return FALSE;
 290
 291     pself->file = last_file;
 292
 293     return TRUE;
 294 }
 295 /* }}} */
 296
 297 /* Convert an object name into a file number, assuming the given prefix
 298  * length. Returns -1 if the object name is invalid, or 0 if the object name
 299  * is a "special" key. */
 300 static int key_to_file(guint prefix_len, const char * key) {
 301     int file;
 302     int i;
 303
 304     /* skip the prefix */
 305     g_return_val_if_fail(strlen(key) > prefix_len, -1);
 306
 307     key += prefix_len;
 308
 309     if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
 310         return 0;
 311     }
 312
 313     /* check that key starts with 'f' */
 314     g_return_val_if_fail(key[0] == 'f', -1);
 315     key++;
 316
 317     /* check that key is of the form "%08x-" */
 318     for (i = 0; i < 8; i++) {
 319         if (!(key[i] >= '0' && key[i] <= '9') &&
 320             !(key[i] >= 'a' && key[i] <= 'f') &&
 321             !(key[i] >= 'A' && key[i] <= 'F')) break;
 322     }
 323     if (key[i] != '-') return -1;
 324     if (i < 8) return -1;
 325
 326     /* convert the file number */
 327     errno = 0;
 328     file = strtoul(key, NULL, 16);
 329     if (errno != 0) {
 330         g_warning(_("unparseable file number '%s'"), key);
 331         return -1;
 332     }
 333
 334     return file;
 335 }
 336
 337 /* {{{ find_last_file */
 338 /* Find the number of the last file that contains any data (even just a header).
 339  * Returns -1 in event of an error
 340  */
 341 static int
 342 find_last_file(S3Device *self) {
 343     gboolean result;
 344     GSList *keys;
 345     unsigned int prefix_len = strlen(self->prefix);
 346     int last_file = 0;
 347
 348     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 349     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 350     if (!result) {
 351         fprintf(stderr, _("While listing S3 keys: %s\n"),
 352                 s3_strerror(self->s3));
 353         return -1;
 354     }
 355
 356     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 357         int file = key_to_file(prefix_len, keys->data);
 358
 359         /* and if it's the last, keep it */
 360         if (file > last_file)
 361             last_file = file;
 362     }
 363
 364     return last_file;
 365 }
 366 /* }}} */
 367
 368 /* {{{ find_next_file */
 369 /* Find the number of the file following the requested one, if any.
 370  * Returns 0 if there is no such file or -1 in event of an error
 371  */
 372 static int
 373 find_next_file(S3Device *self, int last_file) {
 374     gboolean result;
 375     GSList *keys;
 376     unsigned int prefix_len = strlen(self->prefix);
 377     int next_file = 0;
 378
 379     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 380     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 381     if (!result) {
 382         fprintf(stderr, _("While listing S3 keys: %s\n"),
 383                 s3_strerror(self->s3));
 384         return -1;
 385     }
 386
 387     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 388         int file;
 389
 390         file = key_to_file(prefix_len, (char*)keys->data);
 391
 392         if (file < 0) {
 393             /* Set this in case we don't find a next file; this is not a
 394              * hard error, so if we can find a next file we'll return that
 395              * instead. */
 396             next_file = -1;
 397         }
 398
 399         if (file < next_file && file > last_file) {
 400             next_file = file;
 401         }
 402     }
 403
 404     return last_file;
 405 }
 406 /* }}} */
 407
 408 /* {{{ delete_file */
 409 static gboolean
 410 delete_file(S3Device *self,
 411             int file)
 412 {
 413     gboolean result;
 414     GSList *keys;
 415     char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
 416
 417     result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
 418     if (!result) {
 419         fprintf(stderr, _("While listing S3 keys: %s\n"),
 420                 s3_strerror(self->s3));
 421         return FALSE;
 422     }
 423
 424     /* this will likely be a *lot* of keys */
 425     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 426         if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
 427         if (!s3_delete(self->s3, self->bucket, keys->data)) {
 428             fprintf(stderr, _("While deleting key '%s': %s\n"),
 429                     (char*)keys->data, s3_strerror(self->s3));
 430             g_slist_free(keys);
 431             return FALSE;
 432         }
 433     }
 434
 435     return TRUE;
 436 }
 437 /* }}} */
 438
 439 /*
 440  * Class mechanics
 441  */
 442
 443 /* {{{ s3_device_register */
 444 void
 445 s3_device_register(void)
 446 {
 447     static const char * device_prefix_list[] = { "s3", NULL };
 448     g_assert(s3_init());
 449     register_device(s3_device_factory, device_prefix_list);
 450 }
 451 /* }}} */
 452
 453 /* {{{ s3_device_get_type */
 454 GType
 455 s3_device_get_type(void)
 456 {
 457     static GType type = 0;
 458
 459     if G_UNLIKELY(type == 0) {
 460         static const GTypeInfo info = {
 461             sizeof (S3DeviceClass),
 462             (GBaseInitFunc) NULL,
 463             (GBaseFinalizeFunc) NULL,
 464             (GClassInitFunc) s3_device_class_init,
 465             (GClassFinalizeFunc) NULL,
 466             NULL /* class_data */,
 467             sizeof (S3Device),
 468             0 /* n_preallocs */,
 469             (GInstanceInitFunc) s3_device_init,
 470             NULL
 471         };
 472
 473         type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
 474                                        (GTypeFlags)0);
 475     }
 476
 477     return type;
 478 }
 479 /* }}} */
 480
 481 /* {{{ s3_device_init */
 482 static void
 483 s3_device_init(S3Device * self)
 484 {
 485     Device * o;
 486     DeviceProperty prop;
 487     GValue response;
 488
 489     self->initializing = TRUE;
 490
 491     /* Register property values */
 492     o = (Device*)(self);
 493     bzero(&response, sizeof(response));
 494
 495     prop.base = &device_property_concurrency;
 496     prop.access = PROPERTY_ACCESS_GET_MASK;
 497     g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
 498     g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
 499     device_add_property(o, &prop, &response);
 500     g_value_unset(&response);
 501
 502     prop.base = &device_property_streaming;
 503     g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
 504     g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
 505     device_add_property(o, &prop, &response);
 506     g_value_unset(&response);
 507
 508     prop.base = &device_property_block_size;
 509     g_value_init(&response, G_TYPE_INT);
 510     g_value_set_int(&response, -1); /* indicates a variable block size; see below */
 511     device_add_property(o, &prop, &response);
 512     g_value_unset(&response);
 513
 514     prop.base = &device_property_min_block_size;
 515     g_value_init(&response, G_TYPE_UINT);
 516     g_value_set_uint(&response, S3_DEVICE_MIN_BLOCK_SIZE);
 517     device_add_property(o, &prop, &response);
 518
 519     prop.base = &device_property_max_block_size;
 520     g_value_set_uint(&response, S3_DEVICE_MAX_BLOCK_SIZE);
 521     device_add_property(o, &prop, &response);
 522     g_value_unset(&response);
 523
 524     prop.base = &device_property_appendable;
 525     g_value_init(&response, G_TYPE_BOOLEAN);
 526     g_value_set_boolean(&response, TRUE);
 527     device_add_property(o, &prop, &response);
 528
 529     prop.base = &device_property_partial_deletion;
 530     g_value_set_boolean(&response, TRUE);
 531     device_add_property(o, &prop, &response);
 532     g_value_unset(&response);
 533
 534     prop.base = &device_property_canonical_name;
 535     g_value_init(&response, G_TYPE_STRING);
 536     g_value_set_static_string(&response, "s3:");
 537     device_add_property(o, &prop, &response);
 538     g_value_unset(&response);
 539
 540     prop.base = &device_property_medium_access_type;
 541     g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
 542     g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
 543     device_add_property(o, &prop, &response);
 544     g_value_unset(&response);
 545
 546     prop.access = PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START;
 547     prop.base = &device_property_s3_secret_key;
 548     device_add_property(o, &prop, NULL);
 549     prop.base = &device_property_s3_access_key;
 550     device_add_property(o, &prop, NULL);
 551 #ifdef WANT_DEVPAY
 552     prop.base = &device_property_s3_user_token;
 553     device_add_property(o, &prop, NULL);
 554 #endif
 555 }
 556 /* }}} */
 557
 558 /* {{{ s3_device_class_init */
 559 static void
 560 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
 561 {
 562     GObjectClass *g_object_class = (GObjectClass*) c;
 563     DeviceClass *device_class = (DeviceClass *)c;
 564
 565     parent_class = g_type_class_ref (TYPE_DEVICE);
 566
 567     device_class->open_device = s3_device_open_device;
 568     device_class->read_label = s3_device_read_label;
 569     device_class->start = s3_device_start;
 570
 571     device_class->start_file = s3_device_start_file;
 572     device_class->write_block = s3_device_write_block;
 573     device_class->finish_file = s3_device_finish_file;
 574
 575     device_class->seek_file = s3_device_seek_file;
 576     device_class->seek_block = s3_device_seek_block;
 577     device_class->read_block = s3_device_read_block;
 578     device_class->recycle_file = s3_device_recycle_file;
 579
 580     device_class->property_set = s3_device_property_set;
 581     device_class->property_get = s3_device_property_get;
 582
 583     g_object_class->finalize = s3_device_finalize;
 584 }
 585 /* }}} */
 586
 587 /* {{{ s3_device_factory */
 588 static Device*
 589 s3_device_factory(char * device_type,
 590                   char * device_name)
 591 {
 592     Device *rval;
 593     S3Device * s3_rval;
 594     g_assert(0 == strcmp(device_type, "s3"));
 595     rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
 596     s3_rval = (S3Device*)rval;
 597
 598     if (!device_open_device(rval, device_name)) {
 599         g_object_unref(rval);
 600         return NULL;
 601     } else {
 602         s3_rval->initializing = FALSE;
 603         return rval;
 604     }
 605
 606 }
 607 /* }}} */
 608
 609 /*
 610  * Virtual function overrides
 611  */
 612
 613 /* {{{ s3_device_open_device */
 614 static gboolean
 615 s3_device_open_device(Device *pself,
 616                       char *device_name)
 617 {
 618     S3Device *self = S3_DEVICE(pself);
 619     char * name_colon;
 620
 621     g_return_val_if_fail(self != NULL, FALSE);
 622
 623     /* Device name may be bucket/prefix, to support multiple volumes in a
 624      * single bucket. */
 625     name_colon = index(device_name, '/');
 626     if (name_colon == NULL) {
 627         self->bucket = g_strdup(device_name);
 628         self->prefix = g_strdup("");
 629     } else {
 630         self->bucket = g_strndup(device_name, name_colon - device_name);
 631         self->prefix = g_strdup(name_colon + 1);
 632     }
 633
 634     if (self->bucket == NULL || self->bucket[0] == '\0') {
 635         fprintf(stderr, _("Empty bucket name in device %s.\n"), device_name);
 636         amfree(self->bucket);
 637         amfree(self->prefix);
 638         return FALSE;
 639     }
 640
 641     g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
 642
 643     /* default value */
 644     self->verbose = FALSE;
 645
 646     if (parent_class->open_device) {
 647         parent_class->open_device(pself, device_name);
 648     }
 649
 650     return TRUE;
 651 }
 652 /* }}} */
 653
 654 /* {{{ s3_device_finalize */
 655 static void s3_device_finalize(GObject * obj_self) {
 656     S3Device *self = S3_DEVICE (obj_self);
 657
 658     if(G_OBJECT_CLASS(parent_class)->finalize)
 659         (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
 660
 661     if(self->s3) s3_free(self->s3);
 662     if(self->bucket) g_free(self->bucket);
 663     if(self->prefix) g_free(self->prefix);
 664 }
 665 /* }}} */
 666
 667 static gboolean setup_handle(S3Device * self, G_GNUC_UNUSED gboolean silent) {
 668     if (self->s3 == NULL) {
 669         if (self->access_key == NULL) {
 670             if (!silent) fprintf(stderr, _("No S3 access key specified\n"));
 671             return FALSE;
 672         }
 673         if (self->secret_key == NULL) {
 674             if (!silent) fprintf(stderr, _("No S3 secret key specified\n"));
 675             return FALSE;
 676         }
 677 #ifdef WANT_DEVPAY
 678         if (self->user_token == NULL) {
 679             if (!silent) fprintf(stderr, _("No S3 user token specified\n"));
 680             return FALSE;
 681         }
 682 #endif
 683         self->s3 = s3_open(self->access_key, self->secret_key
 684 #ifdef WANT_DEVPAY
 685                            , self->user_token
 686 #endif
 687                            );
 688         if (self->s3 == NULL) {
 689             fprintf(stderr, "Internal error creating S3 handle.\n");
 690             return FALSE;
 691         }
 692     }
 693
 694     s3_verbose(self->s3, self->verbose);
 695
 696     return TRUE;
 697 }
 698
 699 /* {{{ s3_device_read_label */
 700 static ReadLabelStatusFlags
 701 s3_device_read_label(Device *pself) {
 702     S3Device *self = S3_DEVICE(pself);
 703     char *key;
 704     gpointer buf;
 705     guint buf_size;
 706     dumpfile_t amanda_header;
 707
 708     if (!setup_handle(self, self->initializing))
 709         return READ_LABEL_STATUS_DEVICE_ERROR;
 710
 711     key = special_file_to_key(self, "tapestart", -1);
 712     if (!s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE)) {
 713         guint response_code;
 714         s3_error_code_t s3_error_code;
 715         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
 716
 717         /* if it's an expected error (not found), just return FALSE */
 718         if (response_code == 404 &&
 719              (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
 720             g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
 721             return READ_LABEL_STATUS_VOLUME_UNLABELED;
 722         }
 723
 724         /* otherwise, log it and return */
 725         fprintf(stderr, _("While trying to read tapestart header: %s\n"),
 726                 s3_strerror(self->s3));
 727         return READ_LABEL_STATUS_DEVICE_ERROR;
 728     }
 729
 730     g_assert(buf != NULL);
 731     fh_init(&amanda_header);
 732     parse_file_header(buf, &amanda_header, buf_size);
 733
 734     g_free(buf);
 735
 736     if (amanda_header.type != F_TAPESTART) {
 737         fprintf(stderr, _("Invalid amanda header\n"));
 738         return READ_LABEL_STATUS_VOLUME_ERROR;
 739     }
 740
 741     amfree(pself->volume_label);
 742     pself->volume_label = g_strdup(amanda_header.name);
 743     amfree(pself->volume_time);
 744     pself->volume_time = g_strdup(amanda_header.datestamp);
 745
 746     return READ_LABEL_STATUS_SUCCESS;
 747 }
 748 /* }}} */
 749
 750 /* {{{ s3_device_start */
 751 static gboolean
 752 s3_device_start (Device * pself, DeviceAccessMode mode,
 753                  char * label, char * timestamp) {
 754     S3Device * self;
 755     int file, last_file;
 756
 757     self = S3_DEVICE(pself);
 758     g_return_val_if_fail (self != NULL, FALSE);
 759
 760     if (!setup_handle(self, FALSE))
 761         return FALSE;
 762
 763     /* try creating the bucket, in case it doesn't exist */
 764     if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
 765         guint response_code;
 766         s3_error_code_t s3_error_code;
 767         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
 768
 769         /* if it isn't an expected error (bucket already exists),
 770          * return FALSE */
 771         if (response_code != 409 ||
 772             s3_error_code != S3_ERROR_BucketAlreadyExists) {
 773             fprintf(stderr, _("While creating new S3 bucket: %s\n"),
 774                     s3_strerror(self->s3));
 775             return FALSE;
 776         }
 777     }
 778
 779     /* call up to the parent (Device) to set access_mode, volume_label,
 780      * and volume_time, either from the arguments (ACCESS_WRITE) or by
 781      * reading from the 0th file (otherwise)
 782      */
 783     if (parent_class->start)
 784         if (!parent_class->start((Device*)self, mode, label, timestamp))
 785             return FALSE;
 786
 787     /* take care of any dirty work for this mode */
 788     switch (mode) {
 789         case ACCESS_READ:
 790             break;
 791
 792         case ACCESS_WRITE:
 793             /* delete all files */
 794             last_file = find_last_file(self);
 795             if (last_file < 0) return FALSE;
 796             for (file = 0; file <= last_file; file++) {
 797                 if (!delete_file(self, file)) return FALSE;
 798             }
 799
 800             /* write a new amanda header */
 801             if (!write_amanda_header(self, label, timestamp)) {
 802                 return FALSE;
 803             }
 804             break;
 805
 806         case ACCESS_APPEND:
 807             return seek_to_end(self);
 808             break;
 809         case ACCESS_NULL:
 810             g_assert_not_reached();
 811     }
 812
 813     g_assert(pself->access_mode == mode);
 814
 815     return TRUE;
 816 }
 817 /* }}} */
 818
 819 static gboolean s3_device_property_get(Device * p_self, DevicePropertyId id,
 820                                        GValue * val) {
 821     S3Device * self;
 822     const DevicePropertyBase * base;
 823
 824     self = S3_DEVICE(p_self);
 825     g_return_val_if_fail(self != NULL, FALSE);
 826
 827     base = device_property_get_by_id(id);
 828     g_return_val_if_fail(self != NULL, FALSE);
 829
 830     g_value_unset_init(val, base->type);
 831
 832     if (id == PROPERTY_S3_SECRET_KEY) {
 833         if (self->secret_key != NULL) {
 834             g_value_set_string(val, self->secret_key);
 835             return TRUE;
 836         } else {
 837             return FALSE;
 838         }
 839     } else if (id == PROPERTY_S3_ACCESS_KEY) {
 840         if (self->access_key != NULL) {
 841             g_value_set_string(val, self->access_key);
 842             return TRUE;
 843         } else {
 844             return FALSE;
 845         }
 846     }
 847 #ifdef WANT_DEVPAY
 848     else if (id == PROPERTY_S3_USER_TOKEN) {
 849         if (self->user_token != NULL) {
 850             g_value_set_string(val, self->user_token);
 851             return TRUE;
 852         } else {
 853             return FALSE;
 854         }
 855     }
 856 #endif /* WANT_DEVPAY */
 857     else if (id == PROPERTY_VERBOSE) {
 858         g_value_set_boolean(val, self->verbose);
 859         return TRUE;
 860     } else {
 861         /* chain up */
 862         if (parent_class->property_get) {
 863             return (parent_class->property_get)(p_self, id, val);
 864         } else {
 865             return FALSE;
 866         }
 867     }
 868
 869     g_assert_not_reached();
 870 }
 871
 872 static gboolean s3_device_property_set(Device * p_self, DevicePropertyId id,
 873                                        GValue * val) {
 874     S3Device * self;
 875     const DevicePropertyBase * base;
 876
 877     self = S3_DEVICE(p_self);
 878     g_return_val_if_fail(self != NULL, FALSE);
 879
 880     base = device_property_get_by_id(id);
 881     g_return_val_if_fail(self != NULL, FALSE);
 882
 883     g_return_val_if_fail(G_VALUE_HOLDS(val, base->type), FALSE);
 884
 885     if (id == PROPERTY_S3_SECRET_KEY) {
 886         if (p_self->access_mode != ACCESS_NULL)
 887             return FALSE;
 888         amfree(self->secret_key);
 889         self->secret_key = g_value_dup_string(val);
 890         device_clear_volume_details(p_self);
 891         return TRUE;
 892     } else if (id == PROPERTY_S3_ACCESS_KEY) {
 893         if (p_self->access_mode != ACCESS_NULL)
 894             return FALSE;
 895         amfree(self->access_key);
 896         self->access_key = g_value_dup_string(val);
 897         device_clear_volume_details(p_self);
 898         return TRUE;
 899     }
 900 #ifdef WANT_DEVPAY
 901     else if (id == PROPERTY_S3_USER_TOKEN) {
 902         if (p_self->access_mode != ACCESS_NULL)
 903             return FALSE;
 904         amfree(self->user_token);
 905         self->user_token = g_value_dup_string(val);
 906         device_clear_volume_details(p_self);
 907         return TRUE;
 908     }
 909 #endif /* WANT_DEVPAY */
 910     else if (id == PROPERTY_VERBOSE) {
 911         self->verbose = g_value_get_boolean(val);
 912         /* Our S3 handle may not yet have been instantiated; if so, it will
 913          * get the proper verbose setting when it is created */
 914         if (self->s3)
 915             s3_verbose(self->s3, self->verbose);
 916         return TRUE;
 917     } else {
 918         if (parent_class->property_set) {
 919             return (parent_class->property_set)(p_self, id, val);
 920         } else {
 921             return FALSE;
 922         }
 923     }
 924
 925     g_assert_not_reached();
 926 }
 927
 928 /* functions for writing */
 929
 930 /* {{{ s3_device_start_file */
 931
 932 static gboolean
 933 s3_device_start_file (Device *pself, const dumpfile_t *jobInfo) {
 934     S3Device *self = S3_DEVICE(pself);
 935     char *amanda_header;
 936     int header_size;
 937     gboolean header_fits, result;
 938     char *key;
 939
 940     g_return_val_if_fail (self != NULL, FALSE);
 941
 942     /* Build the amanda header. */
 943     amanda_header = device_build_amanda_header(pself, jobInfo,
 944                                                &header_size, &header_fits);
 945     g_return_val_if_fail(amanda_header != NULL, FALSE);
 946     g_return_val_if_fail(header_fits, FALSE);
 947
 948     /* set the file and block numbers correctly */
 949     pself->file = (pself->file > 0)? pself->file+1 : 1;
 950     pself->block = 0;
 951     pself->in_file = TRUE;
 952
 953     /* write it out as a special block (not the 0th) */
 954     key = special_file_to_key(self, "filestart", pself->file);
 955     result = s3_upload(self->s3, self->bucket, key, amanda_header, header_size);
 956     g_free(amanda_header);
 957     g_free(key);
 958     if (!result) {
 959         fprintf(stderr, _("While writing filestart header: %s\n"),
 960                 s3_strerror(self->s3));
 961         return FALSE;
 962     }
 963
 964     return TRUE;
 965 }
 966 /* }}} */
 967
 968 /* {{{ s3_device_write_block */
 969 static gboolean
 970 s3_device_write_block (Device * pself, guint size, gpointer data,
 971                          gboolean last_block) {
 972     gboolean result;
 973     char *filename;
 974     S3Device * self = S3_DEVICE(pself);;
 975
 976     g_assert (self != NULL);
 977     g_assert (data != NULL);
 978
 979     filename = file_and_block_to_key(self, pself->file, pself->block);
 980
 981     result = s3_upload(self->s3, self->bucket, filename, data, size);
 982     g_free(filename);
 983     if (!result) {
 984         fprintf(stderr, _("While writing data block to S3: %s\n"),
 985                 s3_strerror(self->s3));
 986         return FALSE;
 987     }
 988
 989     pself->block++;
 990
 991     /* if this is the last block, finish the file */
 992     if (last_block) {
 993         return s3_device_finish_file(pself);
 994     }
 995
 996     return TRUE;
 997 }
 998 /* }}} */
 999
1000 /* {{{ s3_device_finish_file */
1001 static gboolean
1002 s3_device_finish_file (Device * pself) {
1003     /* we're not in a file anymore */
1004     pself->in_file = FALSE;
1005
1006     return TRUE;
1007 }
1008 /* }}} */
1009
1010 /* {{{ s3_device_recycle_file */
1011 static gboolean
1012 s3_device_recycle_file(Device *pself, guint file) {
1013     S3Device *self = S3_DEVICE(pself);
1014
1015     return delete_file(self, file);
1016 }
1017 /* }}} */
1018
1019 /* functions for reading */
1020
1021 /* {{{ s3_device_seek_file */
1022 static dumpfile_t*
1023 s3_device_seek_file(Device *pself, guint file) {
1024     S3Device *self = S3_DEVICE(pself);
1025     gboolean result;
1026     char *key;
1027     gpointer buf;
1028     guint buf_size;
1029     dumpfile_t *amanda_header;
1030
1031     pself->file = file;
1032     pself->block = 0;
1033     pself->in_file = TRUE;
1034
1035     /* read it in */
1036     key = special_file_to_key(self, "filestart", pself->file);
1037     result = s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE);
1038     g_free(key);
1039
1040     if (!result) {
1041         guint response_code;
1042         s3_error_code_t s3_error_code;
1043         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1044
1045         /* if it's an expected error (not found), check what to do. */
1046         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1047             int next_file;
1048             pself->file = -1;
1049             pself->in_file = FALSE;
1050             next_file = find_next_file(self, pself->file);
1051             if (next_file > 0) {
1052                 /* Note short-circut of dispatcher. */
1053                 return s3_device_seek_file(pself, next_file);
1054             } else if (next_file == 0) {
1055                 /* No next file. Check if we are one past the end. */
1056                 key = special_file_to_key(self, "filestart", pself->file - 1);
1057                 result = s3_read(self->s3, self->bucket, key, &buf, &buf_size,
1058                                  S3_DEVICE_MAX_BLOCK_SIZE);
1059                 g_free(key);
1060                 if (result) {
1061                     return make_tapeend_header();
1062                 } else {
1063                     return NULL;
1064                 }
1065             }
1066         } else {
1067             /* An error occured finding out if we are the last file. */
1068             return NULL;
1069         }
1070     }
1071
1072     /* and make a dumpfile_t out of it */
1073     g_assert(buf != NULL);
1074     amanda_header = g_new(dumpfile_t, 1);
1075     fh_init(amanda_header);
1076     parse_file_header(buf, amanda_header, buf_size);
1077     g_free(buf);
1078
1079     switch (amanda_header->type) {
1080         case F_DUMPFILE:
1081         case F_CONT_DUMPFILE:
1082         case F_SPLIT_DUMPFILE:
1083             return amanda_header;
1084
1085         default:
1086             fprintf(stderr,
1087                     _("Invalid amanda header while reading file header\n"));
1088             g_free(amanda_header);
1089             return NULL;
1090     }
1091 }
1092 /* }}} */
1093
1094 /* {{{ s3_device_seek_block */
1095 static gboolean
1096 s3_device_seek_block(Device *pself, guint64 block) {
1097     pself->block = block;
1098     return TRUE;
1099 }
1100 /* }}} */
1101
1102 /* {{{ s3_device_read_block */
1103 static int
1104 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1105     S3Device * self = S3_DEVICE(pself);
1106     char *key;
1107     gpointer buf;
1108     gboolean result;
1109     guint buf_size;
1110
1111     g_assert (self != NULL);
1112
1113     /* get the file*/
1114     key = file_and_block_to_key(self, pself->file, pself->block);
1115     g_assert(key != NULL);
1116     if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1117         /* use the cached copy and clear the cache */
1118         buf = self->cached_buf;
1119         buf_size = self->cached_size;
1120
1121         self->cached_buf = NULL;
1122         g_free(self->cached_key);
1123         self->cached_key = NULL;
1124     } else {
1125         /* clear the cache and actually download the file */
1126         if (self->cached_buf) {
1127             g_free(self->cached_buf);
1128             self->cached_buf = NULL;
1129         }
1130         if (self->cached_key) {
1131             g_free(self->cached_key);
1132             self->cached_key = NULL;
1133         }
1134
1135         result = s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE);
1136         if (!result) {
1137             guint response_code;
1138             s3_error_code_t s3_error_code;
1139             s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1140
1141             g_free(key);
1142             key = NULL;
1143
1144             /* if it's an expected error (not found), just return -1 */
1145             if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1146                 pself->is_eof = TRUE;
1147                 pself->in_file = FALSE;
1148                 return -1;
1149             }
1150
1151             /* otherwise, log it and return FALSE */
1152             fprintf(stderr, _("While reading data block from S3: %s\n"),
1153                     s3_strerror(self->s3));
1154             return -1;
1155         }
1156     }
1157
1158     /* INVARIANT: cache is NULL */
1159     g_assert(self->cached_buf == NULL);
1160     g_assert(self->cached_key == NULL);
1161
1162     /* now see how the caller wants to deal with that */
1163     if (data == NULL || *size_req < 0 || buf_size > (guint)*size_req) {
1164         /* A size query or short buffer -- load the cache and return the size*/
1165         self->cached_buf = buf;
1166         self->cached_key = key;
1167         self->cached_size = buf_size;
1168
1169         *size_req = buf_size;
1170         return 0;
1171     } else {
1172         /* ok, all checks are passed -- copy the data */
1173         *size_req = buf_size;
1174         g_memmove(data, buf, buf_size);
1175         g_free(key);
1176         g_free(buf);
1177
1178         /* move on to the next block */
1179         pself->block++;
1180
1181         return buf_size;
1182     }
1183 }
1184 /* }}} */