git.gag.com Git - debian/amanda/blob - device-src/s3-device.c

   1 /*
   2  * Copyright (c) 2008, 2009, 2010 Zmanda, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 as published
   6  * by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  10  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  16  *
  17  * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
  18  * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
  19  */
  20
  21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
  22  * data.  It stores data in keys named with a user-specified prefix, inside a
  23  * user-specified bucket.  Data is stored in the form of numbered (large)
  24  * blocks.
  25  */
  26
  27 #include "amanda.h"
  28 #include <string.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <dirent.h>
  33 #include <regex.h>
  34 #include <time.h>
  35 #include "util.h"
  36 #include "conffile.h"
  37 #include "device.h"
  38 #include "s3.h"
  39 #include <curl/curl.h>
  40 #ifdef HAVE_OPENSSL_HMAC_H
  41 # include <openssl/hmac.h>
  42 #else
  43 # ifdef HAVE_CRYPTO_HMAC_H
  44 #  include <crypto/hmac.h>
  45 # else
  46 #  ifdef HAVE_HMAC_H
  47 #   include <hmac.h>
  48 #  endif
  49 # endif
  50 #endif
  51
  52 /*
  53  * Type checking and casting macros
  54  */
  55 #define TYPE_S3_DEVICE  (s3_device_get_type())
  56 #define S3_DEVICE(obj)  G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
  57 #define S3_DEVICE_CONST(obj)    G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
  58 #define S3_DEVICE_CLASS(klass)  G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
  59 #define IS_S3_DEVICE(obj)       G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
  60
  61 #define S3_DEVICE_GET_CLASS(obj)        G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
  62 static GType    s3_device_get_type      (void);
  63
  64 /*
  65  * Main object structure
  66  */
  67 typedef struct _S3MetadataFile S3MetadataFile;
  68
  69 typedef struct _S3Device S3Device;
  70 struct _S3Device {
  71     Device __parent__;
  72
  73     /* The "easy" curl handle we use to access Amazon S3 */
  74     S3Handle *s3;
  75
  76     /* S3 access information */
  77     char *bucket;
  78     char *prefix;
  79
  80     /* The S3 access information. */
  81     char *secret_key;
  82     char *access_key;
  83     char *user_token;
  84
  85     char *bucket_location;
  86     char *storage_class;
  87
  88     char *ca_info;
  89
  90     /* a cache for unsuccessful reads (where we get the file but the caller
  91      * doesn't have space for it or doesn't want it), where we expect the
  92      * next call will request the same file.
  93      */
  94     char *cached_buf;
  95     char *cached_key;
  96     int cached_size;
  97
  98     /* Produce verbose output? */
  99     gboolean verbose;
 100
 101     /* Use SSL? */
 102     gboolean use_ssl;
 103
 104     /* Throttling */
 105     guint64 max_send_speed;
 106     guint64 max_recv_speed;
 107 };
 108
 109 /*
 110  * Class definition
 111  */
 112 typedef struct _S3DeviceClass S3DeviceClass;
 113 struct _S3DeviceClass {
 114     DeviceClass __parent__;
 115 };
 116
 117
 118 /*
 119  * Constants and static data
 120  */
 121
 122 #define S3_DEVICE_NAME "s3"
 123
 124 /* Maximum key length as specified in the S3 documentation
 125  * (*excluding* null terminator) */
 126 #define S3_MAX_KEY_LENGTH 1024
 127
 128 /* Note: for compatability, min can only be decreased and max increased */
 129 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
 130 #define S3_DEVICE_MAX_BLOCK_SIZE (100*1024*1024)
 131 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
 132
 133 /* This goes in lieu of file number for metadata. */
 134 #define SPECIAL_INFIX "special-"
 135
 136 /* pointer to the class of our parent */
 137 static DeviceClass *parent_class = NULL;
 138
 139 /*
 140  * device-specific properties
 141  */
 142
 143 /* Authentication information for Amazon S3. Both of these are strings. */
 144 static DevicePropertyBase device_property_s3_access_key;
 145 static DevicePropertyBase device_property_s3_secret_key;
 146 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
 147 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
 148
 149 /* Same, but for S3 with DevPay. */
 150 static DevicePropertyBase device_property_s3_user_token;
 151 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
 152
 153 /* Location constraint for new buckets created on Amazon S3. */
 154 static DevicePropertyBase device_property_s3_bucket_location;
 155 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
 156
 157 /* Storage class */
 158 static DevicePropertyBase device_property_s3_storage_class;
 159 #define PROPERTY_S3_STORAGE_CLASS (device_property_s3_storage_class.ID)
 160
 161 /* Path to certificate authority certificate */
 162 static DevicePropertyBase device_property_ssl_ca_info;
 163 #define PROPERTY_SSL_CA_INFO (device_property_ssl_ca_info.ID)
 164
 165 /* Whether to use SSL with Amazon S3. */
 166 static DevicePropertyBase device_property_s3_ssl;
 167 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
 168
 169 /* Speed limits for sending and receiving */
 170 static DevicePropertyBase device_property_max_send_speed;
 171 static DevicePropertyBase device_property_max_recv_speed;
 172 #define PROPERTY_MAX_SEND_SPEED (device_property_max_send_speed.ID)
 173 #define PROPERTY_MAX_RECV_SPEED (device_property_max_recv_speed.ID)
 174
 175
 176 /*
 177  * prototypes
 178  */
 179
 180 void s3_device_register(void);
 181
 182 /*
 183  * utility functions */
 184
 185 /* Given file and block numbers, return an S3 key.
 186  *
 187  * @param self: the S3Device object
 188  * @param file: the file number
 189  * @param block: the block within that file
 190  * @returns: a newly allocated string containing an S3 key.
 191  */
 192 static char *
 193 file_and_block_to_key(S3Device *self,
 194                       int file,
 195                       guint64 block);
 196
 197 /* Given the name of a special file (such as 'tapestart'), generate
 198  * the S3 key to use for that file.
 199  *
 200  * @param self: the S3Device object
 201  * @param special_name: name of the special file
 202  * @param file: a file number to include; omitted if -1
 203  * @returns: a newly alocated string containing an S3 key.
 204  */
 205 static char *
 206 special_file_to_key(S3Device *self,
 207                     char *special_name,
 208                     int file);
 209 /* Write an amanda header file to S3.
 210  *
 211  * @param self: the S3Device object
 212  * @param label: the volume label
 213  * @param timestamp: the volume timestamp
 214  */
 215 static gboolean
 216 write_amanda_header(S3Device *self,
 217                     char *label,
 218                     char * timestamp);
 219
 220 /* "Fast forward" this device to the end by looking up the largest file number
 221  * present and setting the current file number one greater.
 222  *
 223  * @param self: the S3Device object
 224  */
 225 static gboolean
 226 seek_to_end(S3Device *self);
 227
 228 /* Find the number of the last file that contains any data (even just a header).
 229  *
 230  * @param self: the S3Device object
 231  * @returns: the last file, or -1 in event of an error
 232  */
 233 static int
 234 find_last_file(S3Device *self);
 235
 236 /* Delete all blocks in the given file, including the filestart block
 237  *
 238  * @param self: the S3Device object
 239  * @param file: the file to delete
 240  */
 241 static gboolean
 242 delete_file(S3Device *self,
 243             int file);
 244
 245
 246 /* Delete all files in the given device
 247  *
 248  * @param self: the S3Device object
 249  */
 250 static gboolean
 251 delete_all_files(S3Device *self);
 252
 253 /* Set up self->s3 as best as possible.
 254  *
 255  * The return value is TRUE iff self->s3 is useable.
 256  *
 257  * @param self: the S3Device object
 258  * @returns: TRUE if the handle is set up
 259  */
 260 static gboolean
 261 setup_handle(S3Device * self);
 262
 263 /*
 264  * class mechanics */
 265
 266 static void
 267 s3_device_init(S3Device * o);
 268
 269 static void
 270 s3_device_class_init(S3DeviceClass * c);
 271
 272 static void
 273 s3_device_finalize(GObject * o);
 274
 275 static Device*
 276 s3_device_factory(char * device_name, char * device_type, char * device_node);
 277
 278 /*
 279  * Property{Get,Set}Fns */
 280
 281 static gboolean s3_device_set_access_key_fn(Device *self,
 282     DevicePropertyBase *base, GValue *val,
 283     PropertySurety surety, PropertySource source);
 284
 285 static gboolean s3_device_set_secret_key_fn(Device *self,
 286     DevicePropertyBase *base, GValue *val,
 287     PropertySurety surety, PropertySource source);
 288
 289 static gboolean s3_device_set_user_token_fn(Device *self,
 290     DevicePropertyBase *base, GValue *val,
 291     PropertySurety surety, PropertySource source);
 292
 293 static gboolean s3_device_set_bucket_location_fn(Device *self,
 294     DevicePropertyBase *base, GValue *val,
 295     PropertySurety surety, PropertySource source);
 296
 297 static gboolean s3_device_set_storage_class_fn(Device *self,
 298     DevicePropertyBase *base, GValue *val,
 299     PropertySurety surety, PropertySource source);
 300
 301 static gboolean s3_device_set_ca_info_fn(Device *self,
 302     DevicePropertyBase *base, GValue *val,
 303     PropertySurety surety, PropertySource source);
 304
 305 static gboolean s3_device_set_verbose_fn(Device *self,
 306     DevicePropertyBase *base, GValue *val,
 307     PropertySurety surety, PropertySource source);
 308
 309 static gboolean s3_device_set_ssl_fn(Device *self,
 310     DevicePropertyBase *base, GValue *val,
 311     PropertySurety surety, PropertySource source);
 312
 313 static gboolean s3_device_set_max_send_speed_fn(Device *self,
 314     DevicePropertyBase *base, GValue *val,
 315     PropertySurety surety, PropertySource source);
 316
 317 static gboolean s3_device_set_max_recv_speed_fn(Device *self,
 318     DevicePropertyBase *base, GValue *val,
 319     PropertySurety surety, PropertySource source);
 320
 321 /*
 322  * virtual functions */
 323
 324 static void
 325 s3_device_open_device(Device *pself, char *device_name,
 326                   char * device_type, char * device_node);
 327
 328 static DeviceStatusFlags s3_device_read_label(Device * self);
 329
 330 static gboolean
 331 s3_device_start(Device * self,
 332                 DeviceAccessMode mode,
 333                 char * label,
 334                 char * timestamp);
 335
 336 static gboolean
 337 s3_device_finish(Device * self);
 338
 339 static gboolean
 340 s3_device_start_file(Device * self,
 341                      dumpfile_t * jobInfo);
 342
 343 static gboolean
 344 s3_device_write_block(Device * self,
 345                       guint size,
 346                       gpointer data);
 347
 348 static gboolean
 349 s3_device_finish_file(Device * self);
 350
 351 static dumpfile_t*
 352 s3_device_seek_file(Device *pself,
 353                     guint file);
 354
 355 static gboolean
 356 s3_device_seek_block(Device *pself,
 357                      guint64 block);
 358
 359 static int
 360 s3_device_read_block(Device * pself,
 361                      gpointer data,
 362                      int *size_req);
 363
 364 static gboolean
 365 s3_device_recycle_file(Device *pself,
 366                        guint file);
 367
 368 static gboolean
 369 s3_device_erase(Device *pself);
 370
 371 /*
 372  * Private functions
 373  */
 374
 375 static char *
 376 file_and_block_to_key(S3Device *self,
 377                       int file,
 378                       guint64 block)
 379 {
 380     char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
 381                                    self->prefix, file, (long long unsigned int)block);
 382     g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
 383     return s3_key;
 384 }
 385
 386 static char *
 387 special_file_to_key(S3Device *self,
 388                     char *special_name,
 389                     int file)
 390 {
 391     if (file == -1)
 392         return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
 393     else
 394         return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
 395 }
 396
 397 static gboolean
 398 write_amanda_header(S3Device *self,
 399                     char *label,
 400                     char * timestamp)
 401 {
 402     CurlBuffer amanda_header = {NULL, 0, 0, 0};
 403     char * key = NULL;
 404     gboolean result;
 405     dumpfile_t * dumpinfo = NULL;
 406     Device *d_self = DEVICE(self);
 407     size_t header_size;
 408
 409     /* build the header */
 410     header_size = 0; /* no minimum size */
 411     dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
 412     amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
 413         &header_size);
 414     if (amanda_header.buffer == NULL) {
 415         device_set_error(d_self,
 416             stralloc(_("Amanda tapestart header won't fit in a single block!")),
 417             DEVICE_STATUS_DEVICE_ERROR);
 418         dumpfile_free(dumpinfo);
 419         g_free(amanda_header.buffer);
 420         return FALSE;
 421     }
 422
 423     /* write out the header and flush the uploads. */
 424     key = special_file_to_key(self, "tapestart", -1);
 425     g_assert(header_size < G_MAXUINT); /* for cast to guint */
 426     amanda_header.buffer_len = (guint)header_size;
 427     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
 428                        &amanda_header, NULL, NULL);
 429     g_free(amanda_header.buffer);
 430     g_free(key);
 431
 432     if (!result) {
 433         device_set_error(d_self,
 434             vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3)),
 435             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 436         dumpfile_free(dumpinfo);
 437     } else {
 438         dumpfile_free(d_self->volume_header);
 439         d_self->volume_header = dumpinfo;
 440     }
 441
 442     return result;
 443 }
 444
 445 static gboolean
 446 seek_to_end(S3Device *self) {
 447     int last_file;
 448
 449     Device *pself = DEVICE(self);
 450
 451     last_file = find_last_file(self);
 452     if (last_file < 0)
 453         return FALSE;
 454
 455     pself->file = last_file;
 456
 457     return TRUE;
 458 }
 459
 460 /* Convert an object name into a file number, assuming the given prefix
 461  * length. Returns -1 if the object name is invalid, or 0 if the object name
 462  * is a "special" key. */
 463 static int key_to_file(guint prefix_len, const char * key) {
 464     int file;
 465     int i;
 466
 467     /* skip the prefix */
 468     if (strlen(key) <= prefix_len)
 469         return -1;
 470
 471     key += prefix_len;
 472
 473     if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
 474         return 0;
 475     }
 476
 477     /* check that key starts with 'f' */
 478     if (key[0] != 'f')
 479         return -1;
 480     key++;
 481
 482     /* check that key is of the form "%08x-" */
 483     for (i = 0; i < 8; i++) {
 484         if (!(key[i] >= '0' && key[i] <= '9') &&
 485             !(key[i] >= 'a' && key[i] <= 'f') &&
 486             !(key[i] >= 'A' && key[i] <= 'F')) break;
 487     }
 488     if (key[i] != '-') return -1;
 489     if (i < 8) return -1;
 490
 491     /* convert the file number */
 492     errno = 0;
 493     file = strtoul(key, NULL, 16);
 494     if (errno != 0) {
 495         g_warning(_("unparseable file number '%s'"), key);
 496         return -1;
 497     }
 498
 499     return file;
 500 }
 501
 502 /* Find the number of the last file that contains any data (even just a header).
 503  * Returns -1 in event of an error
 504  */
 505 static int
 506 find_last_file(S3Device *self) {
 507     gboolean result;
 508     GSList *keys;
 509     unsigned int prefix_len = strlen(self->prefix);
 510     int last_file = 0;
 511     Device *d_self = DEVICE(self);
 512
 513     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 514     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 515     if (!result) {
 516         device_set_error(d_self,
 517             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 518             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 519         return -1;
 520     }
 521
 522     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 523         int file = key_to_file(prefix_len, keys->data);
 524
 525         /* and if it's the last, keep it */
 526         if (file > last_file)
 527             last_file = file;
 528     }
 529
 530     return last_file;
 531 }
 532
 533 /* Find the number of the file following the requested one, if any.
 534  * Returns 0 if there is no such file or -1 in event of an error
 535  */
 536 static int
 537 find_next_file(S3Device *self, int last_file) {
 538     gboolean result;
 539     GSList *keys;
 540     unsigned int prefix_len = strlen(self->prefix);
 541     int next_file = 0;
 542     Device *d_self = DEVICE(self);
 543
 544     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 545     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 546     if (!result) {
 547         device_set_error(d_self,
 548             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 549             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 550         return -1;
 551     }
 552
 553     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 554         int file;
 555
 556         file = key_to_file(prefix_len, (char*)keys->data);
 557
 558         if (file < 0) {
 559             /* Set this in case we don't find a next file; this is not a
 560              * hard error, so if we can find a next file we'll return that
 561              * instead. */
 562             next_file = -1;
 563         }
 564
 565         if (file < next_file && file > last_file) {
 566             next_file = file;
 567         }
 568     }
 569
 570     return next_file;
 571 }
 572
 573 static gboolean
 574 delete_file(S3Device *self,
 575             int file)
 576 {
 577     gboolean result;
 578     GSList *keys;
 579     char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
 580     Device *d_self = DEVICE(self);
 581
 582     result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
 583     if (!result) {
 584         device_set_error(d_self,
 585             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 586             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 587         return FALSE;
 588     }
 589
 590     /* this will likely be a *lot* of keys */
 591     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 592         if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
 593         if (!s3_delete(self->s3, self->bucket, keys->data)) {
 594             device_set_error(d_self,
 595                 vstrallocf(_("While deleting key '%s': %s"),
 596                             (char*)keys->data, s3_strerror(self->s3)),
 597                 DEVICE_STATUS_DEVICE_ERROR);
 598             g_slist_free(keys);
 599             return FALSE;
 600         }
 601     }
 602
 603     return TRUE;
 604 }
 605
 606 static gboolean
 607 delete_all_files(S3Device *self)
 608 {
 609     int file, last_file;
 610
 611     /*
 612      * Note: this has to be allowed to retry for a while because the bucket
 613      * may have been created and not yet appeared
 614      */
 615     last_file = find_last_file(self);
 616     if (last_file < 0) {
 617         guint response_code;
 618         s3_error_code_t s3_error_code;
 619         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
 620
 621         /*
 622          * if the bucket doesn't exist, it doesn't conatin any files,
 623          * so the operation is a success
 624          */
 625         if ((response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket)) {
 626             /* find_last_file set an error; clear it */
 627             device_set_error(DEVICE(self), NULL, DEVICE_STATUS_SUCCESS);
 628             return TRUE;
 629         } else {
 630             /* find_last_file already set the error */
 631             return FALSE;
 632         }
 633     }
 634
 635     for (file = 1; file <= last_file; file++) {
 636         if (!delete_file(self, file))
 637             /* delete_file already set our error message */
 638             return FALSE;
 639     }
 640
 641     return TRUE;
 642 }
 643
 644 /*
 645  * Class mechanics
 646  */
 647
 648 void
 649 s3_device_register(void)
 650 {
 651     static const char * device_prefix_list[] = { S3_DEVICE_NAME, NULL };
 652     g_assert(s3_init());
 653
 654     /* set up our properties */
 655     device_property_fill_and_register(&device_property_s3_secret_key,
 656                                       G_TYPE_STRING, "s3_secret_key",
 657        "Secret access key to authenticate with Amazon S3");
 658     device_property_fill_and_register(&device_property_s3_access_key,
 659                                       G_TYPE_STRING, "s3_access_key",
 660        "Access key ID to authenticate with Amazon S3");
 661     device_property_fill_and_register(&device_property_s3_user_token,
 662                                       G_TYPE_STRING, "s3_user_token",
 663        "User token for authentication Amazon devpay requests");
 664     device_property_fill_and_register(&device_property_s3_bucket_location,
 665                                       G_TYPE_STRING, "s3_bucket_location",
 666        "Location constraint for buckets on Amazon S3");
 667     device_property_fill_and_register(&device_property_s3_storage_class,
 668                                       G_TYPE_STRING, "s3_storage_class",
 669        "Storage class as specified by Amazon (STANDARD or REDUCED_REDUNDANCY)");
 670     device_property_fill_and_register(&device_property_ssl_ca_info,
 671                                       G_TYPE_STRING, "ssl_ca_info",
 672        "Path to certificate authority certificate");
 673     device_property_fill_and_register(&device_property_s3_ssl,
 674                                       G_TYPE_BOOLEAN, "s3_ssl",
 675        "Whether to use SSL with Amazon S3");
 676     device_property_fill_and_register(&device_property_max_send_speed,
 677                                       G_TYPE_UINT64, "max_send_speed",
 678        "Maximum average upload speed (bytes/sec)");
 679     device_property_fill_and_register(&device_property_max_recv_speed,
 680                                       G_TYPE_UINT64, "max_recv_speed",
 681        "Maximum average download speed (bytes/sec)");
 682
 683     /* register the device itself */
 684     register_device(s3_device_factory, device_prefix_list);
 685 }
 686
 687 static GType
 688 s3_device_get_type(void)
 689 {
 690     static GType type = 0;
 691
 692     if G_UNLIKELY(type == 0) {
 693         static const GTypeInfo info = {
 694             sizeof (S3DeviceClass),
 695             (GBaseInitFunc) NULL,
 696             (GBaseFinalizeFunc) NULL,
 697             (GClassInitFunc) s3_device_class_init,
 698             (GClassFinalizeFunc) NULL,
 699             NULL /* class_data */,
 700             sizeof (S3Device),
 701             0 /* n_preallocs */,
 702             (GInstanceInitFunc) s3_device_init,
 703             NULL
 704         };
 705
 706         type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
 707                                        (GTypeFlags)0);
 708     }
 709
 710     return type;
 711 }
 712
 713 static void
 714 s3_device_init(S3Device * self)
 715 {
 716     Device * dself = DEVICE(self);
 717     GValue response;
 718
 719     /* Register property values
 720      * Note: Some aren't added until s3_device_open_device()
 721      */
 722     bzero(&response, sizeof(response));
 723
 724     g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
 725     g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
 726     device_set_simple_property(dself, PROPERTY_CONCURRENCY,
 727             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 728     g_value_unset(&response);
 729
 730     g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
 731     g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
 732     device_set_simple_property(dself, PROPERTY_STREAMING,
 733             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 734     g_value_unset(&response);
 735
 736     g_value_init(&response, G_TYPE_BOOLEAN);
 737     g_value_set_boolean(&response, TRUE);
 738     device_set_simple_property(dself, PROPERTY_APPENDABLE,
 739             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 740     g_value_unset(&response);
 741
 742     g_value_init(&response, G_TYPE_BOOLEAN);
 743     g_value_set_boolean(&response, TRUE);
 744     device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
 745             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 746     g_value_unset(&response);
 747
 748     g_value_init(&response, G_TYPE_BOOLEAN);
 749     g_value_set_boolean(&response, TRUE);
 750     device_set_simple_property(dself, PROPERTY_FULL_DELETION,
 751             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 752     g_value_unset(&response);
 753
 754     g_value_init(&response, G_TYPE_BOOLEAN);
 755     g_value_set_boolean(&response, TRUE); /* well, there *is* no EOM on S3 .. */
 756     device_set_simple_property(dself, PROPERTY_LEOM,
 757             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 758     g_value_unset(&response);
 759
 760     g_value_init(&response, G_TYPE_BOOLEAN);
 761     g_value_set_boolean(&response, FALSE);
 762     device_set_simple_property(dself, PROPERTY_COMPRESSION,
 763             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 764     g_value_unset(&response);
 765
 766     g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
 767     g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
 768     device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
 769             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 770     g_value_unset(&response);
 771
 772 }
 773
 774 static void
 775 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
 776 {
 777     GObjectClass *g_object_class = (GObjectClass*) c;
 778     DeviceClass *device_class = (DeviceClass *)c;
 779
 780     parent_class = g_type_class_ref (TYPE_DEVICE);
 781
 782     device_class->open_device = s3_device_open_device;
 783     device_class->read_label = s3_device_read_label;
 784     device_class->start = s3_device_start;
 785     device_class->finish = s3_device_finish;
 786
 787     device_class->start_file = s3_device_start_file;
 788     device_class->write_block = s3_device_write_block;
 789     device_class->finish_file = s3_device_finish_file;
 790
 791     device_class->seek_file = s3_device_seek_file;
 792     device_class->seek_block = s3_device_seek_block;
 793     device_class->read_block = s3_device_read_block;
 794     device_class->recycle_file = s3_device_recycle_file;
 795
 796     device_class->erase = s3_device_erase;
 797
 798     g_object_class->finalize = s3_device_finalize;
 799
 800     device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
 801             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 802             device_simple_property_get_fn,
 803             s3_device_set_access_key_fn);
 804
 805     device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
 806             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 807             device_simple_property_get_fn,
 808             s3_device_set_secret_key_fn);
 809
 810     device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
 811             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 812             device_simple_property_get_fn,
 813             s3_device_set_user_token_fn);
 814
 815     device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
 816             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 817             device_simple_property_get_fn,
 818             s3_device_set_bucket_location_fn);
 819
 820     device_class_register_property(device_class, PROPERTY_S3_STORAGE_CLASS,
 821             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 822             device_simple_property_get_fn,
 823             s3_device_set_storage_class_fn);
 824
 825     device_class_register_property(device_class, PROPERTY_SSL_CA_INFO,
 826             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 827             device_simple_property_get_fn,
 828             s3_device_set_ca_info_fn);
 829
 830     device_class_register_property(device_class, PROPERTY_VERBOSE,
 831             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 832             device_simple_property_get_fn,
 833             s3_device_set_verbose_fn);
 834
 835     device_class_register_property(device_class, PROPERTY_S3_SSL,
 836             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 837             device_simple_property_get_fn,
 838             s3_device_set_ssl_fn);
 839
 840     device_class_register_property(device_class, PROPERTY_MAX_SEND_SPEED,
 841             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 842             device_simple_property_get_fn,
 843             s3_device_set_max_send_speed_fn);
 844
 845     device_class_register_property(device_class, PROPERTY_MAX_RECV_SPEED,
 846             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 847             device_simple_property_get_fn,
 848             s3_device_set_max_recv_speed_fn);
 849
 850     device_class_register_property(device_class, PROPERTY_COMPRESSION,
 851             PROPERTY_ACCESS_GET_MASK,
 852             device_simple_property_get_fn,
 853             NULL);
 854 }
 855
 856 static gboolean
 857 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
 858     GValue *val, PropertySurety surety, PropertySource source)
 859 {
 860     S3Device *self = S3_DEVICE(p_self);
 861
 862     amfree(self->access_key);
 863     self->access_key = g_value_dup_string(val);
 864     device_clear_volume_details(p_self);
 865
 866     return device_simple_property_set_fn(p_self, base, val, surety, source);
 867 }
 868
 869 static gboolean
 870 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
 871     GValue *val, PropertySurety surety, PropertySource source)
 872 {
 873     S3Device *self = S3_DEVICE(p_self);
 874
 875     amfree(self->secret_key);
 876     self->secret_key = g_value_dup_string(val);
 877     device_clear_volume_details(p_self);
 878
 879     return device_simple_property_set_fn(p_self, base, val, surety, source);
 880 }
 881
 882 static gboolean
 883 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
 884     GValue *val, PropertySurety surety, PropertySource source)
 885 {
 886     S3Device *self = S3_DEVICE(p_self);
 887
 888     amfree(self->user_token);
 889     self->user_token = g_value_dup_string(val);
 890     device_clear_volume_details(p_self);
 891
 892     return device_simple_property_set_fn(p_self, base, val, surety, source);
 893 }
 894
 895 static gboolean
 896 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
 897     GValue *val, PropertySurety surety, PropertySource source)
 898 {
 899     S3Device *self = S3_DEVICE(p_self);
 900     char *str_val = g_value_dup_string(val);
 901
 902     if (str_val[0] && self->use_ssl && !s3_curl_location_compat()) {
 903         device_set_error(p_self, stralloc(_(
 904                 "Location constraint given for Amazon S3 bucket, "
 905                 "but libcurl is too old support wildcard certificates.")),
 906             DEVICE_STATUS_DEVICE_ERROR);
 907         goto fail;
 908     }
 909
 910     if (str_val[0] && !s3_bucket_location_compat(self->bucket)) {
 911         device_set_error(p_self, g_strdup_printf(_(
 912                 "Location constraint given for Amazon S3 bucket, "
 913                 "but the bucket name (%s) is not usable as a subdomain."),
 914                 self->bucket),
 915             DEVICE_STATUS_DEVICE_ERROR);
 916         goto fail;
 917     }
 918
 919     amfree(self->bucket_location);
 920     self->bucket_location = str_val;
 921     device_clear_volume_details(p_self);
 922
 923     return device_simple_property_set_fn(p_self, base, val, surety, source);
 924 fail:
 925     g_free(str_val);
 926     return FALSE;
 927 }
 928
 929 static gboolean
 930 s3_device_set_storage_class_fn(Device *p_self, DevicePropertyBase *base,
 931     GValue *val, PropertySurety surety, PropertySource source)
 932 {
 933     S3Device *self = S3_DEVICE(p_self);
 934     char *str_val = g_value_dup_string(val);
 935
 936     amfree(self->storage_class);
 937     self->storage_class = str_val;
 938     device_clear_volume_details(p_self);
 939
 940     return device_simple_property_set_fn(p_self, base, val, surety, source);
 941 }
 942
 943 static gboolean
 944 s3_device_set_ca_info_fn(Device *p_self, DevicePropertyBase *base,
 945     GValue *val, PropertySurety surety, PropertySource source)
 946 {
 947     S3Device *self = S3_DEVICE(p_self);
 948
 949     amfree(self->ca_info);
 950     self->ca_info = g_value_dup_string(val);
 951     device_clear_volume_details(p_self);
 952
 953     return device_simple_property_set_fn(p_self, base, val, surety, source);
 954 }
 955
 956 static gboolean
 957 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
 958     GValue *val, PropertySurety surety, PropertySource source)
 959 {
 960     S3Device *self = S3_DEVICE(p_self);
 961
 962     self->verbose = g_value_get_boolean(val);
 963     /* Our S3 handle may not yet have been instantiated; if so, it will
 964      * get the proper verbose setting when it is created */
 965     if (self->s3)
 966         s3_verbose(self->s3, self->verbose);
 967
 968     return device_simple_property_set_fn(p_self, base, val, surety, source);
 969 }
 970
 971 static gboolean
 972 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
 973     GValue *val, PropertySurety surety, PropertySource source)
 974 {
 975     S3Device *self = S3_DEVICE(p_self);
 976     gboolean new_val;
 977
 978     new_val = g_value_get_boolean(val);
 979     /* Our S3 handle may not yet have been instantiated; if so, it will
 980      * get the proper use_ssl setting when it is created */
 981     if (self->s3 && !s3_use_ssl(self->s3, new_val)) {
 982         device_set_error(p_self, g_strdup_printf(_(
 983                 "Error setting S3 SSL/TLS use "
 984                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
 985             DEVICE_STATUS_DEVICE_ERROR);
 986         return FALSE;
 987     }
 988     self->use_ssl = new_val;
 989
 990     return device_simple_property_set_fn(p_self, base, val, surety, source);
 991 }
 992
 993 static gboolean
 994 s3_device_set_max_send_speed_fn(Device *p_self,
 995     DevicePropertyBase *base, GValue *val,
 996     PropertySurety surety, PropertySource source)
 997 {
 998     S3Device *self = S3_DEVICE(p_self);
 999     guint64 new_val;
1000
1001     new_val = g_value_get_uint64(val);
1002     if (self->s3 && !s3_set_max_send_speed(self->s3, new_val)) {
1003         device_set_error(p_self,
1004                 g_strdup("Could not set S3 maximum send speed"),
1005                 DEVICE_STATUS_DEVICE_ERROR);
1006         return FALSE;
1007     }
1008     self->max_send_speed = new_val;
1009
1010     return device_simple_property_set_fn(p_self, base, val, surety, source);
1011 }
1012
1013 static gboolean
1014 s3_device_set_max_recv_speed_fn(Device *p_self,
1015     DevicePropertyBase *base, GValue *val,
1016     PropertySurety surety, PropertySource source)
1017 {
1018     S3Device *self = S3_DEVICE(p_self);
1019     guint64 new_val;
1020
1021     new_val = g_value_get_uint64(val);
1022     if (self->s3 && !s3_set_max_recv_speed(self->s3, new_val)) {
1023         device_set_error(p_self,
1024                 g_strdup("Could not set S3 maximum recv speed"),
1025                 DEVICE_STATUS_DEVICE_ERROR);
1026         return FALSE;
1027     }
1028     self->max_recv_speed = new_val;
1029
1030     return device_simple_property_set_fn(p_self, base, val, surety, source);
1031 }
1032
1033 static Device*
1034 s3_device_factory(char * device_name, char * device_type, char * device_node)
1035 {
1036     Device *rval;
1037     S3Device * s3_rval;
1038     g_assert(0 == strcmp(device_type, S3_DEVICE_NAME));
1039     rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
1040     s3_rval = (S3Device*)rval;
1041
1042     device_open_device(rval, device_name, device_type, device_node);
1043     return rval;
1044 }
1045
1046 /*
1047  * Virtual function overrides
1048  */
1049
1050 static void
1051 s3_device_open_device(Device *pself, char *device_name,
1052                         char * device_type, char * device_node)
1053 {
1054     S3Device *self = S3_DEVICE(pself);
1055     char * name_colon;
1056     GValue tmp_value;
1057
1058     pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
1059     pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
1060     pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
1061
1062     /* Device name may be bucket/prefix, to support multiple volumes in a
1063      * single bucket. */
1064     name_colon = strchr(device_node, '/');
1065     if (name_colon == NULL) {
1066         self->bucket = g_strdup(device_node);
1067         self->prefix = g_strdup("");
1068     } else {
1069         self->bucket = g_strndup(device_node, name_colon - device_node);
1070         self->prefix = g_strdup(name_colon + 1);
1071     }
1072
1073     if (self->bucket == NULL || self->bucket[0] == '\0') {
1074         device_set_error(pself,
1075             vstrallocf(_("Empty bucket name in device %s"), device_name),
1076             DEVICE_STATUS_DEVICE_ERROR);
1077         amfree(self->bucket);
1078         amfree(self->prefix);
1079         return;
1080     }
1081
1082     g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
1083
1084     /* default values */
1085     self->verbose = FALSE;
1086
1087     /* use SSL if available */
1088     self->use_ssl = s3_curl_supports_ssl();
1089     bzero(&tmp_value, sizeof(GValue));
1090     g_value_init(&tmp_value, G_TYPE_BOOLEAN);
1091     g_value_set_boolean(&tmp_value, self->use_ssl);
1092     device_set_simple_property(pself, device_property_s3_ssl.ID,
1093         &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
1094
1095     if (parent_class->open_device) {
1096         parent_class->open_device(pself, device_name, device_type, device_node);
1097     }
1098 }
1099
1100 static void s3_device_finalize(GObject * obj_self) {
1101     S3Device *self = S3_DEVICE (obj_self);
1102
1103     if(G_OBJECT_CLASS(parent_class)->finalize)
1104         (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
1105
1106     if(self->s3) s3_free(self->s3);
1107     if(self->bucket) g_free(self->bucket);
1108     if(self->prefix) g_free(self->prefix);
1109     if(self->access_key) g_free(self->access_key);
1110     if(self->secret_key) g_free(self->secret_key);
1111     if(self->user_token) g_free(self->user_token);
1112     if(self->bucket_location) g_free(self->bucket_location);
1113     if(self->storage_class) g_free(self->storage_class);
1114     if(self->ca_info) g_free(self->ca_info);
1115 }
1116
1117 static gboolean setup_handle(S3Device * self) {
1118     Device *d_self = DEVICE(self);
1119     if (self->s3 == NULL) {
1120
1121         if (self->access_key == NULL || self->access_key[0] == '\0') {
1122             device_set_error(d_self,
1123                 stralloc(_("No Amazon access key specified")),
1124                 DEVICE_STATUS_DEVICE_ERROR);
1125             return FALSE;
1126         }
1127
1128         if (self->secret_key == NULL || self->secret_key[0] == '\0') {
1129             device_set_error(d_self,
1130                 stralloc(_("No Amazon secret key specified")),
1131                 DEVICE_STATUS_DEVICE_ERROR);
1132             return FALSE;
1133         }
1134
1135         if (!self->use_ssl && self->ca_info) {
1136             amfree(self->ca_info);
1137         }
1138
1139         self->s3 = s3_open(self->access_key, self->secret_key, self->user_token,
1140             self->bucket_location, self->storage_class, self->ca_info);
1141         if (self->s3 == NULL) {
1142             device_set_error(d_self,
1143                 stralloc(_("Internal error creating S3 handle")),
1144                 DEVICE_STATUS_DEVICE_ERROR);
1145             return FALSE;
1146         }
1147     }
1148
1149     s3_verbose(self->s3, self->verbose);
1150
1151     if (!s3_use_ssl(self->s3, self->use_ssl)) {
1152         device_set_error(d_self, g_strdup_printf(_(
1153                 "Error setting S3 SSL/TLS use "
1154                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
1155             DEVICE_STATUS_DEVICE_ERROR);
1156         return FALSE;
1157     }
1158
1159     if (self->max_send_speed &&
1160             !s3_set_max_send_speed(self->s3, self->max_send_speed)) {
1161         device_set_error(d_self,
1162                 g_strdup("Could not set S3 maximum send speed"),
1163                 DEVICE_STATUS_DEVICE_ERROR);
1164         return FALSE;
1165     }
1166
1167     if (self->max_recv_speed &&
1168             !s3_set_max_recv_speed(self->s3, self->max_recv_speed)) {
1169         device_set_error(d_self,
1170                 g_strdup("Could not set S3 maximum recv speed"),
1171                 DEVICE_STATUS_DEVICE_ERROR);
1172         return FALSE;
1173     }
1174
1175     return TRUE;
1176 }
1177
1178 static DeviceStatusFlags
1179 s3_device_read_label(Device *pself) {
1180     S3Device *self = S3_DEVICE(pself);
1181     char *key;
1182     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1183     dumpfile_t *amanda_header;
1184
1185     /* note that this may be called from s3_device_start, when
1186      * self->access_mode is not ACCESS_NULL */
1187
1188     amfree(pself->volume_label);
1189     amfree(pself->volume_time);
1190     dumpfile_free(pself->volume_header);
1191     pself->volume_header = NULL;
1192
1193     if (device_in_error(self)) return pself->status;
1194
1195     if (!setup_handle(self)) {
1196         /* setup_handle already set our error message */
1197         return pself->status;
1198     }
1199
1200     key = special_file_to_key(self, "tapestart", -1);
1201     if (!s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
1202         guint response_code;
1203         s3_error_code_t s3_error_code;
1204         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1205
1206         /* if it's an expected error (not found), just return FALSE */
1207         if (response_code == 404 &&
1208              (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
1209             g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
1210             device_set_error(pself,
1211                 stralloc(_("Amanda header not found -- unlabeled volume?")),
1212                   DEVICE_STATUS_DEVICE_ERROR
1213                 | DEVICE_STATUS_VOLUME_ERROR
1214                 | DEVICE_STATUS_VOLUME_UNLABELED);
1215             return pself->status;
1216         }
1217
1218         /* otherwise, log it and return */
1219         device_set_error(pself,
1220             vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3)),
1221             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1222         return pself->status;
1223     }
1224
1225     /* handle an empty file gracefully */
1226     if (buf.buffer_len == 0) {
1227         device_set_error(pself, stralloc(_("Empty header file")), DEVICE_STATUS_VOLUME_ERROR);
1228         return pself->status;
1229     }
1230
1231     g_assert(buf.buffer != NULL);
1232     amanda_header = g_new(dumpfile_t, 1);
1233     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1234     pself->volume_header = amanda_header;
1235     g_free(buf.buffer);
1236
1237     if (amanda_header->type != F_TAPESTART) {
1238         device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
1239         return pself->status;
1240     }
1241
1242     pself->volume_label = g_strdup(amanda_header->name);
1243     pself->volume_time = g_strdup(amanda_header->datestamp);
1244     /* pself->volume_header is already set */
1245
1246     device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1247
1248     return pself->status;
1249 }
1250
1251 static gboolean
1252 s3_device_start (Device * pself, DeviceAccessMode mode,
1253                  char * label, char * timestamp) {
1254     S3Device * self;
1255
1256     self = S3_DEVICE(pself);
1257
1258     if (device_in_error(self)) return FALSE;
1259
1260     if (!setup_handle(self)) {
1261         /* setup_handle already set our error message */
1262         return FALSE;
1263     }
1264
1265     pself->access_mode = mode;
1266     pself->in_file = FALSE;
1267
1268     /* try creating the bucket, in case it doesn't exist */
1269     if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
1270         guint response_code;
1271         s3_error_code_t s3_error_code;
1272         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1273
1274         /* if it isn't an expected error (bucket already exists),
1275          * return FALSE */
1276         if (response_code != 409 ||
1277             s3_error_code != S3_ERROR_BucketAlreadyExists) {
1278             device_set_error(pself,
1279                 vstrallocf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3)),
1280                 DEVICE_STATUS_DEVICE_ERROR);
1281             return FALSE;
1282         }
1283     }
1284
1285     /* take care of any dirty work for this mode */
1286     switch (mode) {
1287         case ACCESS_READ:
1288             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1289                 /* s3_device_read_label already set our error message */
1290                 return FALSE;
1291             }
1292             break;
1293
1294         case ACCESS_WRITE:
1295             delete_all_files(self);
1296
1297             /* write a new amanda header */
1298             if (!write_amanda_header(self, label, timestamp)) {
1299                 return FALSE;
1300             }
1301
1302             pself->volume_label = newstralloc(pself->volume_label, label);
1303             pself->volume_time = newstralloc(pself->volume_time, timestamp);
1304
1305             /* unset the VOLUME_UNLABELED flag, if it was set */
1306             device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1307             break;
1308
1309         case ACCESS_APPEND:
1310             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1311                 /* s3_device_read_label already set our error message */
1312                 return FALSE;
1313             }
1314             return seek_to_end(self);
1315             break;
1316
1317         case ACCESS_NULL:
1318             g_assert_not_reached();
1319     }
1320
1321     return TRUE;
1322 }
1323
1324 static gboolean
1325 s3_device_finish (Device * pself) {
1326     /* we're not in a file anymore */
1327     pself->access_mode = ACCESS_NULL;
1328
1329     if (device_in_error(pself)) return FALSE;
1330
1331     return TRUE;
1332 }
1333
1334 /* functions for writing */
1335
1336
1337 static gboolean
1338 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
1339     S3Device *self = S3_DEVICE(pself);
1340     CurlBuffer amanda_header = {NULL, 0, 0, 0};
1341     gboolean result;
1342     size_t header_size;
1343     char *key;
1344
1345     if (device_in_error(self)) return FALSE;
1346
1347     pself->is_eom = FALSE;
1348
1349     /* Set the blocksize to zero, since there's no header to skip (it's stored
1350      * in a distinct file, rather than block zero) */
1351     jobInfo->blocksize = 0;
1352
1353     /* Build the amanda header. */
1354     header_size = 0; /* no minimum size */
1355     amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
1356         &header_size);
1357     if (amanda_header.buffer == NULL) {
1358         device_set_error(pself,
1359             stralloc(_("Amanda file header won't fit in a single block!")),
1360             DEVICE_STATUS_DEVICE_ERROR);
1361         return FALSE;
1362     }
1363     amanda_header.buffer_len = header_size;
1364
1365     /* set the file and block numbers correctly */
1366     pself->file = (pself->file > 0)? pself->file+1 : 1;
1367     pself->block = 0;
1368     pself->in_file = TRUE;
1369
1370     /* write it out as a special block (not the 0th) */
1371     key = special_file_to_key(self, "filestart", pself->file);
1372     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
1373                        &amanda_header, NULL, NULL);
1374     g_free(amanda_header.buffer);
1375     g_free(key);
1376     if (!result) {
1377         device_set_error(pself,
1378             vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3)),
1379             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1380         return FALSE;
1381     }
1382
1383     return TRUE;
1384 }
1385
1386 static gboolean
1387 s3_device_write_block (Device * pself, guint size, gpointer data) {
1388     gboolean result;
1389     char *filename;
1390     S3Device * self = S3_DEVICE(pself);
1391     CurlBuffer to_write = {data, size, 0, 0};
1392
1393     g_assert (self != NULL);
1394     g_assert (data != NULL);
1395     if (device_in_error(self)) return FALSE;
1396
1397     filename = file_and_block_to_key(self, pself->file, pself->block);
1398
1399     result = s3_upload(self->s3, self->bucket, filename, S3_BUFFER_READ_FUNCS,
1400         &to_write, NULL, NULL);
1401     g_free(filename);
1402     if (!result) {
1403         device_set_error(pself,
1404             vstrallocf(_("While writing data block to S3: %s"), s3_strerror(self->s3)),
1405             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1406         return FALSE;
1407     }
1408
1409     pself->block++;
1410
1411     return TRUE;
1412 }
1413
1414 static gboolean
1415 s3_device_finish_file (Device * pself) {
1416     if (device_in_error(pself)) return FALSE;
1417
1418     /* we're not in a file anymore */
1419     pself->in_file = FALSE;
1420
1421     return TRUE;
1422 }
1423
1424 static gboolean
1425 s3_device_recycle_file(Device *pself, guint file) {
1426     S3Device *self = S3_DEVICE(pself);
1427     if (device_in_error(self)) return FALSE;
1428
1429     return delete_file(self, file);
1430     /* delete_file already set our error message if necessary */
1431 }
1432
1433 static gboolean
1434 s3_device_erase(Device *pself) {
1435     S3Device *self = S3_DEVICE(pself);
1436     char *key = NULL;
1437     const char *errmsg = NULL;
1438     guint response_code;
1439     s3_error_code_t s3_error_code;
1440
1441     if (!setup_handle(self)) {
1442         /* error set by setup_handle */
1443         return FALSE;
1444     }
1445
1446     key = special_file_to_key(self, "tapestart", -1);
1447     if (!s3_delete(self->s3, self->bucket, key)) {
1448         s3_error(self->s3, &errmsg, NULL, NULL, NULL, NULL, NULL);
1449         device_set_error(pself,
1450             stralloc(errmsg),
1451             DEVICE_STATUS_DEVICE_ERROR);
1452         return FALSE;
1453     }
1454     g_free(key);
1455
1456     if (!delete_all_files(self))
1457         return FALSE;
1458
1459     if (!s3_delete_bucket(self->s3, self->bucket)) {
1460         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1461
1462         /*
1463          * ignore the error if the bucket isn't empty (there may be data from elsewhere)
1464          * or the bucket not existing (already deleted perhaps?)
1465          */
1466         if (!(
1467                 (response_code == 409 && s3_error_code == S3_ERROR_BucketNotEmpty) ||
1468                 (response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket))) {
1469
1470             device_set_error(pself,
1471                 stralloc(errmsg),
1472                 DEVICE_STATUS_DEVICE_ERROR);
1473             return FALSE;
1474         }
1475     }
1476     return TRUE;
1477 }
1478
1479 /* functions for reading */
1480
1481 static dumpfile_t*
1482 s3_device_seek_file(Device *pself, guint file) {
1483     S3Device *self = S3_DEVICE(pself);
1484     gboolean result;
1485     char *key;
1486     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1487     dumpfile_t *amanda_header;
1488     const char *errmsg = NULL;
1489
1490     if (device_in_error(self)) return NULL;
1491
1492     pself->file = file;
1493     pself->is_eof = FALSE;
1494     pself->in_file = FALSE;
1495     pself->block = 0;
1496
1497     /* read it in */
1498     key = special_file_to_key(self, "filestart", pself->file);
1499     result = s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
1500         &buf, NULL, NULL);
1501     g_free(key);
1502
1503     if (!result) {
1504         guint response_code;
1505         s3_error_code_t s3_error_code;
1506         s3_error(self->s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
1507
1508         /* if it's an expected error (not found), check what to do. */
1509         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1510             int next_file;
1511             next_file = find_next_file(self, pself->file);
1512             if (next_file > 0) {
1513                 /* Note short-circut of dispatcher. */
1514                 return s3_device_seek_file(pself, next_file);
1515             } else if (next_file == 0) {
1516                 /* No next file. Check if we are one past the end. */
1517                 key = special_file_to_key(self, "filestart", pself->file - 1);
1518                 result = s3_read(self->s3, self->bucket, key,
1519                     S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
1520                 g_free(key);
1521                 if (result) {
1522                     /* pself->file, etc. are already correct */
1523                     return make_tapeend_header();
1524                 } else {
1525                     device_set_error(pself,
1526                         stralloc(_("Attempt to read past tape-end file")),
1527                         DEVICE_STATUS_SUCCESS);
1528                     return NULL;
1529                 }
1530             }
1531         } else {
1532             /* An unexpected error occured finding out if we are the last file. */
1533             device_set_error(pself,
1534                 stralloc(errmsg),
1535                 DEVICE_STATUS_DEVICE_ERROR);
1536             return NULL;
1537         }
1538     }
1539
1540     /* and make a dumpfile_t out of it */
1541     g_assert(buf.buffer != NULL);
1542     amanda_header = g_new(dumpfile_t, 1);
1543     fh_init(amanda_header);
1544     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1545     g_free(buf.buffer);
1546
1547     switch (amanda_header->type) {
1548         case F_DUMPFILE:
1549         case F_CONT_DUMPFILE:
1550         case F_SPLIT_DUMPFILE:
1551             break;
1552
1553         default:
1554             device_set_error(pself,
1555                 stralloc(_("Invalid amanda header while reading file header")),
1556                 DEVICE_STATUS_VOLUME_ERROR);
1557             g_free(amanda_header);
1558             return NULL;
1559     }
1560
1561     pself->in_file = TRUE;
1562     return amanda_header;
1563 }
1564
1565 static gboolean
1566 s3_device_seek_block(Device *pself, guint64 block) {
1567     if (device_in_error(pself)) return FALSE;
1568
1569     pself->block = block;
1570     return TRUE;
1571 }
1572
1573 typedef struct s3_read_block_data {
1574     gpointer data;
1575     int size_req;
1576     int size_written;
1577
1578     CurlBuffer curl;
1579 } s3_read_block_data;
1580
1581 /* wrapper around s3_buffer_write_func to write as much data as possible to
1582  * the user's buffer, and switch to a dynamically allocated buffer if that
1583  * isn't large enough */
1584 static size_t
1585 s3_read_block_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
1586 {
1587     s3_read_block_data *dat = stream;
1588     guint new_bytes, bytes_needed;
1589
1590     /* if data is NULL, call through to s3_buffer_write_func */
1591     if (!dat->data) {
1592         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1593     }
1594
1595     new_bytes = (guint) size * nmemb;
1596     bytes_needed = dat->size_written + new_bytes;
1597
1598     if (bytes_needed > (guint)dat->size_req) {
1599         /* this read will overflow the user's buffer, so malloc ourselves
1600          * a new buffer and keep reading */
1601         dat->curl.buffer = g_malloc(bytes_needed);
1602         dat->curl.buffer_len = bytes_needed;
1603         dat->curl.buffer_pos = dat->size_written;
1604         memcpy(dat->curl.buffer, dat->data, dat->size_written);
1605         dat->data = NULL; /* signal that the user's buffer is too small */
1606         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1607     }
1608
1609     /* copy it into the dat->data buffer, and increment the size */
1610     memcpy(dat->data + dat->size_written, ptr, new_bytes);
1611     dat->size_written += new_bytes;
1612
1613     return new_bytes;
1614 }
1615
1616 static int
1617 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1618     S3Device * self = S3_DEVICE(pself);
1619     char *key;
1620     s3_read_block_data dat = {NULL, 0, 0, { NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE} };
1621     gboolean result;
1622
1623     g_assert (self != NULL);
1624     if (device_in_error(self)) return -1;
1625
1626     /* get the file*/
1627     key = file_and_block_to_key(self, pself->file, pself->block);
1628     g_assert(key != NULL);
1629     if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1630         if (*size_req >= self->cached_size) {
1631             /* use the cached copy and clear the cache */
1632             memcpy(data, self->cached_buf, self->cached_size);
1633             *size_req = self->cached_size;
1634
1635             g_free(key);
1636             g_free(self->cached_key);
1637             self->cached_key = NULL;
1638             g_free(self->cached_buf);
1639             self->cached_buf = NULL;
1640
1641             pself->block++;
1642             return *size_req;
1643         } else {
1644             *size_req = self->cached_size;
1645             g_free(key);
1646             return 0;
1647         }
1648     }
1649
1650     /* clear the cache, as it's useless to us */
1651     if (self->cached_key) {
1652         g_free(self->cached_key);
1653         self->cached_key = NULL;
1654
1655         g_free(self->cached_buf);
1656         self->cached_buf = NULL;
1657     }
1658
1659     /* set up dat for the write_func callback */
1660     if (!data || *size_req <= 0) {
1661         dat.data = NULL;
1662         dat.size_req = 0;
1663     } else {
1664         dat.data = data;
1665         dat.size_req = *size_req;
1666     }
1667
1668     result = s3_read(self->s3, self->bucket, key, s3_read_block_write_func,
1669         s3_buffer_reset_func, &dat, NULL, NULL);
1670     if (!result) {
1671         guint response_code;
1672         s3_error_code_t s3_error_code;
1673         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1674
1675         g_free(key);
1676         key = NULL;
1677
1678         /* if it's an expected error (not found), just return -1 */
1679         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1680             pself->is_eof = TRUE;
1681             pself->in_file = FALSE;
1682             device_set_error(pself,
1683                 stralloc(_("EOF")),
1684                 DEVICE_STATUS_SUCCESS);
1685             return -1;
1686         }
1687
1688         /* otherwise, log it and return FALSE */
1689         device_set_error(pself,
1690             vstrallocf(_("While reading data block from S3: %s"), s3_strerror(self->s3)),
1691             DEVICE_STATUS_VOLUME_ERROR);
1692         return -1;
1693     }
1694
1695     if (dat.data == NULL) {
1696         /* data was larger than the available space, so cache it and return
1697          * the actual size */
1698         self->cached_buf = dat.curl.buffer;
1699         self->cached_size = dat.curl.buffer_pos;
1700         self->cached_key = key;
1701         key = NULL;
1702
1703         *size_req = dat.curl.buffer_pos;
1704         return 0;
1705     }
1706
1707     /* ok, the read went directly to the user's buffer, so we need only
1708      * set and return the size */
1709     pself->block++;
1710     g_free(key);
1711     *size_req = dat.size_written;
1712     return dat.size_written;
1713 }