git.gag.com Git - debian/amanda/blob - device-src/s3-device.c

   1 /*
   2  * Copyright (c) 2008, 2009, 2010 Zmanda, Inc.  All Rights Reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 as published
   6  * by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  10  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
  16  *
  17  * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
  18  * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
  19  */
  20
  21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
  22  * data.  It stores data in keys named with a user-specified prefix, inside a
  23  * user-specified bucket.  Data is stored in the form of numbered (large)
  24  * blocks.
  25  */
  26
  27 #include "amanda.h"
  28 #include <string.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <unistd.h>
  32 #include <dirent.h>
  33 #include <regex.h>
  34 #include <time.h>
  35 #include "util.h"
  36 #include "conffile.h"
  37 #include "device.h"
  38 #include "s3.h"
  39 #include <curl/curl.h>
  40 #ifdef HAVE_OPENSSL_HMAC_H
  41 # include <openssl/hmac.h>
  42 #else
  43 # ifdef HAVE_CRYPTO_HMAC_H
  44 #  include <crypto/hmac.h>
  45 # else
  46 #  ifdef HAVE_HMAC_H
  47 #   include <hmac.h>
  48 #  endif
  49 # endif
  50 #endif
  51
  52 /*
  53  * Type checking and casting macros
  54  */
  55 #define TYPE_S3_DEVICE  (s3_device_get_type())
  56 #define S3_DEVICE(obj)  G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
  57 #define S3_DEVICE_CONST(obj)    G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
  58 #define S3_DEVICE_CLASS(klass)  G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
  59 #define IS_S3_DEVICE(obj)       G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
  60
  61 #define S3_DEVICE_GET_CLASS(obj)        G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
  62 static GType    s3_device_get_type      (void);
  63
  64 /*
  65  * Main object structure
  66  */
  67 typedef struct _S3MetadataFile S3MetadataFile;
  68
  69 typedef struct _S3Device S3Device;
  70 struct _S3Device {
  71     Device __parent__;
  72
  73     /* The "easy" curl handle we use to access Amazon S3 */
  74     S3Handle *s3;
  75
  76     /* S3 access information */
  77     char *bucket;
  78     char *prefix;
  79
  80     /* The S3 access information. */
  81     char *secret_key;
  82     char *access_key;
  83     char *user_token;
  84
  85     char *bucket_location;
  86
  87     char *ca_info;
  88
  89     /* a cache for unsuccessful reads (where we get the file but the caller
  90      * doesn't have space for it or doesn't want it), where we expect the
  91      * next call will request the same file.
  92      */
  93     char *cached_buf;
  94     char *cached_key;
  95     int cached_size;
  96
  97     /* Produce verbose output? */
  98     gboolean verbose;
  99
 100     /* Use SSL? */
 101     gboolean use_ssl;
 102
 103     /* Throttling */
 104     guint64 max_send_speed;
 105     guint64 max_recv_speed;
 106 };
 107
 108 /*
 109  * Class definition
 110  */
 111 typedef struct _S3DeviceClass S3DeviceClass;
 112 struct _S3DeviceClass {
 113     DeviceClass __parent__;
 114 };
 115
 116
 117 /*
 118  * Constants and static data
 119  */
 120
 121 #define S3_DEVICE_NAME "s3"
 122
 123 /* Maximum key length as specified in the S3 documentation
 124  * (*excluding* null terminator) */
 125 #define S3_MAX_KEY_LENGTH 1024
 126
 127 /* Note: for compatability, min can only be decreased and max increased */
 128 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
 129 #define S3_DEVICE_MAX_BLOCK_SIZE (100*1024*1024)
 130 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
 131
 132 /* This goes in lieu of file number for metadata. */
 133 #define SPECIAL_INFIX "special-"
 134
 135 /* pointer to the class of our parent */
 136 static DeviceClass *parent_class = NULL;
 137
 138 /*
 139  * device-specific properties
 140  */
 141
 142 /* Authentication information for Amazon S3. Both of these are strings. */
 143 static DevicePropertyBase device_property_s3_access_key;
 144 static DevicePropertyBase device_property_s3_secret_key;
 145 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
 146 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
 147
 148 /* Same, but for S3 with DevPay. */
 149 static DevicePropertyBase device_property_s3_user_token;
 150 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
 151
 152 /* Location constraint for new buckets created on Amazon S3. */
 153 static DevicePropertyBase device_property_s3_bucket_location;
 154 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
 155
 156 /* Path to certificate authority certificate */
 157 static DevicePropertyBase device_property_ssl_ca_info;
 158 #define PROPERTY_SSL_CA_INFO (device_property_ssl_ca_info.ID)
 159
 160 /* Whether to use SSL with Amazon S3. */
 161 static DevicePropertyBase device_property_s3_ssl;
 162 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
 163
 164 /* Speed limits for sending and receiving */
 165 static DevicePropertyBase device_property_max_send_speed;
 166 static DevicePropertyBase device_property_max_recv_speed;
 167 #define PROPERTY_MAX_SEND_SPEED (device_property_max_send_speed.ID)
 168 #define PROPERTY_MAX_RECV_SPEED (device_property_max_recv_speed.ID)
 169
 170
 171 /*
 172  * prototypes
 173  */
 174
 175 void s3_device_register(void);
 176
 177 /*
 178  * utility functions */
 179
 180 /* Given file and block numbers, return an S3 key.
 181  *
 182  * @param self: the S3Device object
 183  * @param file: the file number
 184  * @param block: the block within that file
 185  * @returns: a newly allocated string containing an S3 key.
 186  */
 187 static char *
 188 file_and_block_to_key(S3Device *self,
 189                       int file,
 190                       guint64 block);
 191
 192 /* Given the name of a special file (such as 'tapestart'), generate
 193  * the S3 key to use for that file.
 194  *
 195  * @param self: the S3Device object
 196  * @param special_name: name of the special file
 197  * @param file: a file number to include; omitted if -1
 198  * @returns: a newly alocated string containing an S3 key.
 199  */
 200 static char *
 201 special_file_to_key(S3Device *self,
 202                     char *special_name,
 203                     int file);
 204 /* Write an amanda header file to S3.
 205  *
 206  * @param self: the S3Device object
 207  * @param label: the volume label
 208  * @param timestamp: the volume timestamp
 209  */
 210 static gboolean
 211 write_amanda_header(S3Device *self,
 212                     char *label,
 213                     char * timestamp);
 214
 215 /* "Fast forward" this device to the end by looking up the largest file number
 216  * present and setting the current file number one greater.
 217  *
 218  * @param self: the S3Device object
 219  */
 220 static gboolean
 221 seek_to_end(S3Device *self);
 222
 223 /* Find the number of the last file that contains any data (even just a header).
 224  *
 225  * @param self: the S3Device object
 226  * @returns: the last file, or -1 in event of an error
 227  */
 228 static int
 229 find_last_file(S3Device *self);
 230
 231 /* Delete all blocks in the given file, including the filestart block
 232  *
 233  * @param self: the S3Device object
 234  * @param file: the file to delete
 235  */
 236 static gboolean
 237 delete_file(S3Device *self,
 238             int file);
 239
 240
 241 /* Delete all files in the given device
 242  *
 243  * @param self: the S3Device object
 244  */
 245 static gboolean
 246 delete_all_files(S3Device *self);
 247
 248 /* Set up self->s3 as best as possible.
 249  *
 250  * The return value is TRUE iff self->s3 is useable.
 251  *
 252  * @param self: the S3Device object
 253  * @returns: TRUE if the handle is set up
 254  */
 255 static gboolean
 256 setup_handle(S3Device * self);
 257
 258 /*
 259  * class mechanics */
 260
 261 static void
 262 s3_device_init(S3Device * o);
 263
 264 static void
 265 s3_device_class_init(S3DeviceClass * c);
 266
 267 static void
 268 s3_device_finalize(GObject * o);
 269
 270 static Device*
 271 s3_device_factory(char * device_name, char * device_type, char * device_node);
 272
 273 /*
 274  * Property{Get,Set}Fns */
 275
 276 static gboolean s3_device_set_access_key_fn(Device *self,
 277     DevicePropertyBase *base, GValue *val,
 278     PropertySurety surety, PropertySource source);
 279
 280 static gboolean s3_device_set_secret_key_fn(Device *self,
 281     DevicePropertyBase *base, GValue *val,
 282     PropertySurety surety, PropertySource source);
 283
 284 static gboolean s3_device_set_user_token_fn(Device *self,
 285     DevicePropertyBase *base, GValue *val,
 286     PropertySurety surety, PropertySource source);
 287
 288 static gboolean s3_device_set_bucket_location_fn(Device *self,
 289     DevicePropertyBase *base, GValue *val,
 290     PropertySurety surety, PropertySource source);
 291
 292 static gboolean s3_device_set_ca_info_fn(Device *self,
 293     DevicePropertyBase *base, GValue *val,
 294     PropertySurety surety, PropertySource source);
 295
 296 static gboolean s3_device_set_verbose_fn(Device *self,
 297     DevicePropertyBase *base, GValue *val,
 298     PropertySurety surety, PropertySource source);
 299
 300 static gboolean s3_device_set_ssl_fn(Device *self,
 301     DevicePropertyBase *base, GValue *val,
 302     PropertySurety surety, PropertySource source);
 303
 304 static gboolean s3_device_set_max_send_speed_fn(Device *self,
 305     DevicePropertyBase *base, GValue *val,
 306     PropertySurety surety, PropertySource source);
 307
 308 static gboolean s3_device_set_max_recv_speed_fn(Device *self,
 309     DevicePropertyBase *base, GValue *val,
 310     PropertySurety surety, PropertySource source);
 311
 312 /*
 313  * virtual functions */
 314
 315 static void
 316 s3_device_open_device(Device *pself, char *device_name,
 317                   char * device_type, char * device_node);
 318
 319 static DeviceStatusFlags s3_device_read_label(Device * self);
 320
 321 static gboolean
 322 s3_device_start(Device * self,
 323                 DeviceAccessMode mode,
 324                 char * label,
 325                 char * timestamp);
 326
 327 static gboolean
 328 s3_device_finish(Device * self);
 329
 330 static gboolean
 331 s3_device_start_file(Device * self,
 332                      dumpfile_t * jobInfo);
 333
 334 static gboolean
 335 s3_device_write_block(Device * self,
 336                       guint size,
 337                       gpointer data);
 338
 339 static gboolean
 340 s3_device_finish_file(Device * self);
 341
 342 static dumpfile_t*
 343 s3_device_seek_file(Device *pself,
 344                     guint file);
 345
 346 static gboolean
 347 s3_device_seek_block(Device *pself,
 348                      guint64 block);
 349
 350 static int
 351 s3_device_read_block(Device * pself,
 352                      gpointer data,
 353                      int *size_req);
 354
 355 static gboolean
 356 s3_device_recycle_file(Device *pself,
 357                        guint file);
 358
 359 static gboolean
 360 s3_device_erase(Device *pself);
 361
 362 /*
 363  * Private functions
 364  */
 365
 366 static char *
 367 file_and_block_to_key(S3Device *self,
 368                       int file,
 369                       guint64 block)
 370 {
 371     char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
 372                                    self->prefix, file, (long long unsigned int)block);
 373     g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
 374     return s3_key;
 375 }
 376
 377 static char *
 378 special_file_to_key(S3Device *self,
 379                     char *special_name,
 380                     int file)
 381 {
 382     if (file == -1)
 383         return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
 384     else
 385         return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
 386 }
 387
 388 static gboolean
 389 write_amanda_header(S3Device *self,
 390                     char *label,
 391                     char * timestamp)
 392 {
 393     CurlBuffer amanda_header = {NULL, 0, 0, 0};
 394     char * key = NULL;
 395     gboolean result;
 396     dumpfile_t * dumpinfo = NULL;
 397     Device *d_self = DEVICE(self);
 398     size_t header_size;
 399
 400     /* build the header */
 401     header_size = 0; /* no minimum size */
 402     dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
 403     amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
 404         &header_size);
 405     if (amanda_header.buffer == NULL) {
 406         device_set_error(d_self,
 407             stralloc(_("Amanda tapestart header won't fit in a single block!")),
 408             DEVICE_STATUS_DEVICE_ERROR);
 409         dumpfile_free(dumpinfo);
 410         g_free(amanda_header.buffer);
 411         return FALSE;
 412     }
 413
 414     /* write out the header and flush the uploads. */
 415     key = special_file_to_key(self, "tapestart", -1);
 416     g_assert(header_size < G_MAXUINT); /* for cast to guint */
 417     amanda_header.buffer_len = (guint)header_size;
 418     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
 419                        &amanda_header, NULL, NULL);
 420     g_free(amanda_header.buffer);
 421     g_free(key);
 422
 423     if (!result) {
 424         device_set_error(d_self,
 425             vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3)),
 426             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 427         dumpfile_free(dumpinfo);
 428     } else {
 429         dumpfile_free(d_self->volume_header);
 430         d_self->volume_header = dumpinfo;
 431     }
 432
 433     return result;
 434 }
 435
 436 static gboolean
 437 seek_to_end(S3Device *self) {
 438     int last_file;
 439
 440     Device *pself = DEVICE(self);
 441
 442     last_file = find_last_file(self);
 443     if (last_file < 0)
 444         return FALSE;
 445
 446     pself->file = last_file;
 447
 448     return TRUE;
 449 }
 450
 451 /* Convert an object name into a file number, assuming the given prefix
 452  * length. Returns -1 if the object name is invalid, or 0 if the object name
 453  * is a "special" key. */
 454 static int key_to_file(guint prefix_len, const char * key) {
 455     int file;
 456     int i;
 457
 458     /* skip the prefix */
 459     if (strlen(key) <= prefix_len)
 460         return -1;
 461
 462     key += prefix_len;
 463
 464     if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
 465         return 0;
 466     }
 467
 468     /* check that key starts with 'f' */
 469     if (key[0] != 'f')
 470         return -1;
 471     key++;
 472
 473     /* check that key is of the form "%08x-" */
 474     for (i = 0; i < 8; i++) {
 475         if (!(key[i] >= '0' && key[i] <= '9') &&
 476             !(key[i] >= 'a' && key[i] <= 'f') &&
 477             !(key[i] >= 'A' && key[i] <= 'F')) break;
 478     }
 479     if (key[i] != '-') return -1;
 480     if (i < 8) return -1;
 481
 482     /* convert the file number */
 483     errno = 0;
 484     file = strtoul(key, NULL, 16);
 485     if (errno != 0) {
 486         g_warning(_("unparseable file number '%s'"), key);
 487         return -1;
 488     }
 489
 490     return file;
 491 }
 492
 493 /* Find the number of the last file that contains any data (even just a header).
 494  * Returns -1 in event of an error
 495  */
 496 static int
 497 find_last_file(S3Device *self) {
 498     gboolean result;
 499     GSList *keys;
 500     unsigned int prefix_len = strlen(self->prefix);
 501     int last_file = 0;
 502     Device *d_self = DEVICE(self);
 503
 504     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 505     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 506     if (!result) {
 507         device_set_error(d_self,
 508             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 509             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 510         return -1;
 511     }
 512
 513     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 514         int file = key_to_file(prefix_len, keys->data);
 515
 516         /* and if it's the last, keep it */
 517         if (file > last_file)
 518             last_file = file;
 519     }
 520
 521     return last_file;
 522 }
 523
 524 /* Find the number of the file following the requested one, if any.
 525  * Returns 0 if there is no such file or -1 in event of an error
 526  */
 527 static int
 528 find_next_file(S3Device *self, int last_file) {
 529     gboolean result;
 530     GSList *keys;
 531     unsigned int prefix_len = strlen(self->prefix);
 532     int next_file = 0;
 533     Device *d_self = DEVICE(self);
 534
 535     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 536     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 537     if (!result) {
 538         device_set_error(d_self,
 539             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 540             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 541         return -1;
 542     }
 543
 544     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 545         int file;
 546
 547         file = key_to_file(prefix_len, (char*)keys->data);
 548
 549         if (file < 0) {
 550             /* Set this in case we don't find a next file; this is not a
 551              * hard error, so if we can find a next file we'll return that
 552              * instead. */
 553             next_file = -1;
 554         }
 555
 556         if (file < next_file && file > last_file) {
 557             next_file = file;
 558         }
 559     }
 560
 561     return next_file;
 562 }
 563
 564 static gboolean
 565 delete_file(S3Device *self,
 566             int file)
 567 {
 568     gboolean result;
 569     GSList *keys;
 570     char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
 571     Device *d_self = DEVICE(self);
 572
 573     result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
 574     if (!result) {
 575         device_set_error(d_self,
 576             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 577             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 578         return FALSE;
 579     }
 580
 581     /* this will likely be a *lot* of keys */
 582     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 583         if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
 584         if (!s3_delete(self->s3, self->bucket, keys->data)) {
 585             device_set_error(d_self,
 586                 vstrallocf(_("While deleting key '%s': %s"),
 587                             (char*)keys->data, s3_strerror(self->s3)),
 588                 DEVICE_STATUS_DEVICE_ERROR);
 589             g_slist_free(keys);
 590             return FALSE;
 591         }
 592     }
 593
 594     return TRUE;
 595 }
 596
 597 static gboolean
 598 delete_all_files(S3Device *self)
 599 {
 600     int file, last_file;
 601
 602     /*
 603      * Note: this has to be allowed to retry for a while because the bucket
 604      * may have been created and not yet appeared
 605      */
 606     last_file = find_last_file(self);
 607     if (last_file < 0) {
 608         guint response_code;
 609         s3_error_code_t s3_error_code;
 610         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
 611
 612         /*
 613          * if the bucket doesn't exist, it doesn't conatin any files,
 614          * so the operation is a success
 615          */
 616         if ((response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket)) {
 617             /* find_last_file set an error; clear it */
 618             device_set_error(DEVICE(self), NULL, DEVICE_STATUS_SUCCESS);
 619             return TRUE;
 620         } else {
 621             /* find_last_file already set the error */
 622             return FALSE;
 623         }
 624     }
 625
 626     for (file = 1; file <= last_file; file++) {
 627         if (!delete_file(self, file))
 628             /* delete_file already set our error message */
 629             return FALSE;
 630     }
 631
 632     return TRUE;
 633 }
 634
 635 /*
 636  * Class mechanics
 637  */
 638
 639 void
 640 s3_device_register(void)
 641 {
 642     static const char * device_prefix_list[] = { S3_DEVICE_NAME, NULL };
 643     g_assert(s3_init());
 644
 645     /* set up our properties */
 646     device_property_fill_and_register(&device_property_s3_secret_key,
 647                                       G_TYPE_STRING, "s3_secret_key",
 648        "Secret access key to authenticate with Amazon S3");
 649     device_property_fill_and_register(&device_property_s3_access_key,
 650                                       G_TYPE_STRING, "s3_access_key",
 651        "Access key ID to authenticate with Amazon S3");
 652     device_property_fill_and_register(&device_property_s3_user_token,
 653                                       G_TYPE_STRING, "s3_user_token",
 654        "User token for authentication Amazon devpay requests");
 655     device_property_fill_and_register(&device_property_s3_bucket_location,
 656                                       G_TYPE_STRING, "s3_bucket_location",
 657        "Location constraint for buckets on Amazon S3");
 658     device_property_fill_and_register(&device_property_ssl_ca_info,
 659                                       G_TYPE_STRING, "ssl_ca_info",
 660        "Path to certificate authority certificate");
 661     device_property_fill_and_register(&device_property_s3_ssl,
 662                                       G_TYPE_BOOLEAN, "s3_ssl",
 663        "Whether to use SSL with Amazon S3");
 664     device_property_fill_and_register(&device_property_max_send_speed,
 665                                       G_TYPE_UINT64, "max_send_speed",
 666        "Maximum average upload speed (bytes/sec)");
 667     device_property_fill_and_register(&device_property_max_recv_speed,
 668                                       G_TYPE_UINT64, "max_recv_speed",
 669        "Maximum average download speed (bytes/sec)");
 670
 671     /* register the device itself */
 672     register_device(s3_device_factory, device_prefix_list);
 673 }
 674
 675 static GType
 676 s3_device_get_type(void)
 677 {
 678     static GType type = 0;
 679
 680     if G_UNLIKELY(type == 0) {
 681         static const GTypeInfo info = {
 682             sizeof (S3DeviceClass),
 683             (GBaseInitFunc) NULL,
 684             (GBaseFinalizeFunc) NULL,
 685             (GClassInitFunc) s3_device_class_init,
 686             (GClassFinalizeFunc) NULL,
 687             NULL /* class_data */,
 688             sizeof (S3Device),
 689             0 /* n_preallocs */,
 690             (GInstanceInitFunc) s3_device_init,
 691             NULL
 692         };
 693
 694         type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
 695                                        (GTypeFlags)0);
 696     }
 697
 698     return type;
 699 }
 700
 701 static void
 702 s3_device_init(S3Device * self)
 703 {
 704     Device * dself = DEVICE(self);
 705     GValue response;
 706
 707     /* Register property values
 708      * Note: Some aren't added until s3_device_open_device()
 709      */
 710     bzero(&response, sizeof(response));
 711
 712     g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
 713     g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
 714     device_set_simple_property(dself, PROPERTY_CONCURRENCY,
 715             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 716     g_value_unset(&response);
 717
 718     g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
 719     g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
 720     device_set_simple_property(dself, PROPERTY_STREAMING,
 721             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 722     g_value_unset(&response);
 723
 724     g_value_init(&response, G_TYPE_BOOLEAN);
 725     g_value_set_boolean(&response, TRUE);
 726     device_set_simple_property(dself, PROPERTY_APPENDABLE,
 727             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 728     g_value_unset(&response);
 729
 730     g_value_init(&response, G_TYPE_BOOLEAN);
 731     g_value_set_boolean(&response, TRUE);
 732     device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
 733             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 734     g_value_unset(&response);
 735
 736     g_value_init(&response, G_TYPE_BOOLEAN);
 737     g_value_set_boolean(&response, TRUE);
 738     device_set_simple_property(dself, PROPERTY_FULL_DELETION,
 739             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 740     g_value_unset(&response);
 741
 742     g_value_init(&response, G_TYPE_BOOLEAN);
 743     g_value_set_boolean(&response, FALSE);
 744     device_set_simple_property(dself, PROPERTY_COMPRESSION,
 745             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 746     g_value_unset(&response);
 747
 748     g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
 749     g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
 750     device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
 751             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 752     g_value_unset(&response);
 753
 754 }
 755
 756 static void
 757 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
 758 {
 759     GObjectClass *g_object_class = (GObjectClass*) c;
 760     DeviceClass *device_class = (DeviceClass *)c;
 761
 762     parent_class = g_type_class_ref (TYPE_DEVICE);
 763
 764     device_class->open_device = s3_device_open_device;
 765     device_class->read_label = s3_device_read_label;
 766     device_class->start = s3_device_start;
 767     device_class->finish = s3_device_finish;
 768
 769     device_class->start_file = s3_device_start_file;
 770     device_class->write_block = s3_device_write_block;
 771     device_class->finish_file = s3_device_finish_file;
 772
 773     device_class->seek_file = s3_device_seek_file;
 774     device_class->seek_block = s3_device_seek_block;
 775     device_class->read_block = s3_device_read_block;
 776     device_class->recycle_file = s3_device_recycle_file;
 777
 778     device_class->erase = s3_device_erase;
 779
 780     g_object_class->finalize = s3_device_finalize;
 781
 782     device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
 783             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 784             device_simple_property_get_fn,
 785             s3_device_set_access_key_fn);
 786
 787     device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
 788             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 789             device_simple_property_get_fn,
 790             s3_device_set_secret_key_fn);
 791
 792     device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
 793             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 794             device_simple_property_get_fn,
 795             s3_device_set_user_token_fn);
 796
 797     device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
 798             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 799             device_simple_property_get_fn,
 800             s3_device_set_bucket_location_fn);
 801
 802     device_class_register_property(device_class, PROPERTY_SSL_CA_INFO,
 803             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 804             device_simple_property_get_fn,
 805             s3_device_set_ca_info_fn);
 806
 807     device_class_register_property(device_class, PROPERTY_VERBOSE,
 808             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 809             device_simple_property_get_fn,
 810             s3_device_set_verbose_fn);
 811
 812     device_class_register_property(device_class, PROPERTY_S3_SSL,
 813             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 814             device_simple_property_get_fn,
 815             s3_device_set_ssl_fn);
 816
 817     device_class_register_property(device_class, PROPERTY_MAX_SEND_SPEED,
 818             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 819             device_simple_property_get_fn,
 820             s3_device_set_max_send_speed_fn);
 821
 822     device_class_register_property(device_class, PROPERTY_MAX_RECV_SPEED,
 823             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 824             device_simple_property_get_fn,
 825             s3_device_set_max_recv_speed_fn);
 826
 827     device_class_register_property(device_class, PROPERTY_COMPRESSION,
 828             PROPERTY_ACCESS_GET_MASK,
 829             device_simple_property_get_fn,
 830             NULL);
 831 }
 832
 833 static gboolean
 834 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
 835     GValue *val, PropertySurety surety, PropertySource source)
 836 {
 837     S3Device *self = S3_DEVICE(p_self);
 838
 839     amfree(self->access_key);
 840     self->access_key = g_value_dup_string(val);
 841     device_clear_volume_details(p_self);
 842
 843     return device_simple_property_set_fn(p_self, base, val, surety, source);
 844 }
 845
 846 static gboolean
 847 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
 848     GValue *val, PropertySurety surety, PropertySource source)
 849 {
 850     S3Device *self = S3_DEVICE(p_self);
 851
 852     amfree(self->secret_key);
 853     self->secret_key = g_value_dup_string(val);
 854     device_clear_volume_details(p_self);
 855
 856     return device_simple_property_set_fn(p_self, base, val, surety, source);
 857 }
 858
 859 static gboolean
 860 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
 861     GValue *val, PropertySurety surety, PropertySource source)
 862 {
 863     S3Device *self = S3_DEVICE(p_self);
 864
 865     amfree(self->user_token);
 866     self->user_token = g_value_dup_string(val);
 867     device_clear_volume_details(p_self);
 868
 869     return device_simple_property_set_fn(p_self, base, val, surety, source);
 870 }
 871
 872 static gboolean
 873 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
 874     GValue *val, PropertySurety surety, PropertySource source)
 875 {
 876     S3Device *self = S3_DEVICE(p_self);
 877     char *str_val = g_value_dup_string(val);
 878
 879     if (str_val[0] && self->use_ssl && !s3_curl_location_compat()) {
 880         device_set_error(p_self, stralloc(_(
 881                 "Location constraint given for Amazon S3 bucket, "
 882                 "but libcurl is too old support wildcard certificates.")),
 883             DEVICE_STATUS_DEVICE_ERROR);
 884         goto fail;
 885     }
 886
 887     if (str_val[0] && !s3_bucket_location_compat(self->bucket)) {
 888         device_set_error(p_self, g_strdup_printf(_(
 889                 "Location constraint given for Amazon S3 bucket, "
 890                 "but the bucket name (%s) is not usable as a subdomain."),
 891                 self->bucket),
 892             DEVICE_STATUS_DEVICE_ERROR);
 893         goto fail;
 894     }
 895
 896     amfree(self->bucket_location);
 897     self->bucket_location = g_value_dup_string(val);
 898     device_clear_volume_details(p_self);
 899
 900     return device_simple_property_set_fn(p_self, base, val, surety, source);
 901 fail:
 902     g_free(str_val);
 903     return FALSE;
 904 }
 905
 906 static gboolean
 907 s3_device_set_ca_info_fn(Device *p_self, DevicePropertyBase *base,
 908     GValue *val, PropertySurety surety, PropertySource source)
 909 {
 910     S3Device *self = S3_DEVICE(p_self);
 911
 912     if (!self->use_ssl) {
 913         device_set_error(p_self, stralloc(_(
 914                 "Path to certificate authority certificate can not be "
 915                 "set if SSL/TLS is not being used.")),
 916             DEVICE_STATUS_DEVICE_ERROR);
 917        return FALSE;
 918     }
 919
 920     amfree(self->ca_info);
 921     self->ca_info = g_value_dup_string(val);
 922     device_clear_volume_details(p_self);
 923
 924     return device_simple_property_set_fn(p_self, base, val, surety, source);
 925 }
 926
 927 static gboolean
 928 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
 929     GValue *val, PropertySurety surety, PropertySource source)
 930 {
 931     S3Device *self = S3_DEVICE(p_self);
 932
 933     self->verbose = g_value_get_boolean(val);
 934     /* Our S3 handle may not yet have been instantiated; if so, it will
 935      * get the proper verbose setting when it is created */
 936     if (self->s3)
 937         s3_verbose(self->s3, self->verbose);
 938
 939     return device_simple_property_set_fn(p_self, base, val, surety, source);
 940 }
 941
 942 static gboolean
 943 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
 944     GValue *val, PropertySurety surety, PropertySource source)
 945 {
 946     S3Device *self = S3_DEVICE(p_self);
 947     gboolean new_val;
 948
 949     new_val = g_value_get_boolean(val);
 950     /* Our S3 handle may not yet have been instantiated; if so, it will
 951      * get the proper use_ssl setting when it is created */
 952     if (self->s3 && !s3_use_ssl(self->s3, new_val)) {
 953         device_set_error(p_self, g_strdup_printf(_(
 954                 "Error setting S3 SSL/TLS use "
 955                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
 956             DEVICE_STATUS_DEVICE_ERROR);
 957         return FALSE;
 958     }
 959     self->use_ssl = new_val;
 960
 961     return device_simple_property_set_fn(p_self, base, val, surety, source);
 962 }
 963
 964 static gboolean
 965 s3_device_set_max_send_speed_fn(Device *p_self,
 966     DevicePropertyBase *base, GValue *val,
 967     PropertySurety surety, PropertySource source)
 968 {
 969     S3Device *self = S3_DEVICE(p_self);
 970     guint64 new_val;
 971
 972     new_val = g_value_get_uint64(val);
 973     if (self->s3 && !s3_set_max_send_speed(self->s3, new_val)) {
 974         device_set_error(p_self,
 975                 g_strdup("Could not set S3 maximum send speed"),
 976                 DEVICE_STATUS_DEVICE_ERROR);
 977         return FALSE;
 978     }
 979     self->max_send_speed = new_val;
 980
 981     return device_simple_property_set_fn(p_self, base, val, surety, source);
 982 }
 983
 984 static gboolean
 985 s3_device_set_max_recv_speed_fn(Device *p_self,
 986     DevicePropertyBase *base, GValue *val,
 987     PropertySurety surety, PropertySource source)
 988 {
 989     S3Device *self = S3_DEVICE(p_self);
 990     guint64 new_val;
 991
 992     new_val = g_value_get_uint64(val);
 993     if (self->s3 && !s3_set_max_recv_speed(self->s3, new_val)) {
 994         device_set_error(p_self,
 995                 g_strdup("Could not set S3 maximum recv speed"),
 996                 DEVICE_STATUS_DEVICE_ERROR);
 997         return FALSE;
 998     }
 999     self->max_recv_speed = new_val;
1000
1001     return device_simple_property_set_fn(p_self, base, val, surety, source);
1002 }
1003
1004 static Device*
1005 s3_device_factory(char * device_name, char * device_type, char * device_node)
1006 {
1007     Device *rval;
1008     S3Device * s3_rval;
1009     g_assert(0 == strcmp(device_type, S3_DEVICE_NAME));
1010     rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
1011     s3_rval = (S3Device*)rval;
1012
1013     device_open_device(rval, device_name, device_type, device_node);
1014     return rval;
1015 }
1016
1017 /*
1018  * Virtual function overrides
1019  */
1020
1021 static void
1022 s3_device_open_device(Device *pself, char *device_name,
1023                         char * device_type, char * device_node)
1024 {
1025     S3Device *self = S3_DEVICE(pself);
1026     char * name_colon;
1027     GValue tmp_value;
1028
1029     pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
1030     pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
1031     pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
1032
1033     /* Device name may be bucket/prefix, to support multiple volumes in a
1034      * single bucket. */
1035     name_colon = strchr(device_node, '/');
1036     if (name_colon == NULL) {
1037         self->bucket = g_strdup(device_node);
1038         self->prefix = g_strdup("");
1039     } else {
1040         self->bucket = g_strndup(device_node, name_colon - device_node);
1041         self->prefix = g_strdup(name_colon + 1);
1042     }
1043
1044     if (self->bucket == NULL || self->bucket[0] == '\0') {
1045         device_set_error(pself,
1046             vstrallocf(_("Empty bucket name in device %s"), device_name),
1047             DEVICE_STATUS_DEVICE_ERROR);
1048         amfree(self->bucket);
1049         amfree(self->prefix);
1050         return;
1051     }
1052
1053     g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
1054
1055     /* default values */
1056     self->verbose = FALSE;
1057
1058     /* use SSL if available */
1059     self->use_ssl = s3_curl_supports_ssl();
1060     bzero(&tmp_value, sizeof(GValue));
1061     g_value_init(&tmp_value, G_TYPE_BOOLEAN);
1062     g_value_set_boolean(&tmp_value, self->use_ssl);
1063     device_set_simple_property(pself, device_property_s3_ssl.ID,
1064         &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
1065
1066     if (parent_class->open_device) {
1067         parent_class->open_device(pself, device_name, device_type, device_node);
1068     }
1069 }
1070
1071 static void s3_device_finalize(GObject * obj_self) {
1072     S3Device *self = S3_DEVICE (obj_self);
1073
1074     if(G_OBJECT_CLASS(parent_class)->finalize)
1075         (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
1076
1077     if(self->s3) s3_free(self->s3);
1078     if(self->bucket) g_free(self->bucket);
1079     if(self->prefix) g_free(self->prefix);
1080     if(self->access_key) g_free(self->access_key);
1081     if(self->secret_key) g_free(self->secret_key);
1082     if(self->user_token) g_free(self->user_token);
1083     if(self->bucket_location) g_free(self->bucket_location);
1084     if(self->ca_info) g_free(self->ca_info);
1085 }
1086
1087 static gboolean setup_handle(S3Device * self) {
1088     Device *d_self = DEVICE(self);
1089     if (self->s3 == NULL) {
1090
1091         if (self->access_key == NULL || self->access_key[0] == '\0') {
1092             device_set_error(d_self,
1093                 stralloc(_("No Amazon access key specified")),
1094                 DEVICE_STATUS_DEVICE_ERROR);
1095             return FALSE;
1096         }
1097
1098         if (self->secret_key == NULL || self->secret_key[0] == '\0') {
1099             device_set_error(d_self,
1100                 stralloc(_("No Amazon secret key specified")),
1101                 DEVICE_STATUS_DEVICE_ERROR);
1102             return FALSE;
1103         }
1104
1105         self->s3 = s3_open(self->access_key, self->secret_key, self->user_token,
1106             self->bucket_location, self->ca_info);
1107         if (self->s3 == NULL) {
1108             device_set_error(d_self,
1109                 stralloc(_("Internal error creating S3 handle")),
1110                 DEVICE_STATUS_DEVICE_ERROR);
1111             return FALSE;
1112         }
1113     }
1114
1115     s3_verbose(self->s3, self->verbose);
1116
1117     if (!s3_use_ssl(self->s3, self->use_ssl)) {
1118         device_set_error(d_self, g_strdup_printf(_(
1119                 "Error setting S3 SSL/TLS use "
1120                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
1121             DEVICE_STATUS_DEVICE_ERROR);
1122         return FALSE;
1123     }
1124
1125     if (self->max_send_speed &&
1126             !s3_set_max_send_speed(self->s3, self->max_send_speed)) {
1127         device_set_error(d_self,
1128                 g_strdup("Could not set S3 maximum send speed"),
1129                 DEVICE_STATUS_DEVICE_ERROR);
1130         return FALSE;
1131     }
1132
1133     if (self->max_recv_speed &&
1134             !s3_set_max_recv_speed(self->s3, self->max_recv_speed)) {
1135         device_set_error(d_self,
1136                 g_strdup("Could not set S3 maximum recv speed"),
1137                 DEVICE_STATUS_DEVICE_ERROR);
1138         return FALSE;
1139     }
1140
1141     return TRUE;
1142 }
1143
1144 static DeviceStatusFlags
1145 s3_device_read_label(Device *pself) {
1146     S3Device *self = S3_DEVICE(pself);
1147     char *key;
1148     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1149     dumpfile_t *amanda_header;
1150
1151     /* note that this may be called from s3_device_start, when
1152      * self->access_mode is not ACCESS_NULL */
1153
1154     amfree(pself->volume_label);
1155     amfree(pself->volume_time);
1156     dumpfile_free(pself->volume_header);
1157     pself->volume_header = NULL;
1158
1159     if (device_in_error(self)) return pself->status;
1160
1161     if (!setup_handle(self)) {
1162         /* setup_handle already set our error message */
1163         return pself->status;
1164     }
1165
1166     key = special_file_to_key(self, "tapestart", -1);
1167     if (!s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
1168         guint response_code;
1169         s3_error_code_t s3_error_code;
1170         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1171
1172         /* if it's an expected error (not found), just return FALSE */
1173         if (response_code == 404 &&
1174              (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
1175             g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
1176             device_set_error(pself,
1177                 stralloc(_("Amanda header not found -- unlabeled volume?")),
1178                   DEVICE_STATUS_DEVICE_ERROR
1179                 | DEVICE_STATUS_VOLUME_ERROR
1180                 | DEVICE_STATUS_VOLUME_UNLABELED);
1181             return pself->status;
1182         }
1183
1184         /* otherwise, log it and return */
1185         device_set_error(pself,
1186             vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3)),
1187             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1188         return pself->status;
1189     }
1190
1191     /* handle an empty file gracefully */
1192     if (buf.buffer_len == 0) {
1193         device_set_error(pself, stralloc(_("Empty header file")), DEVICE_STATUS_VOLUME_ERROR);
1194         return pself->status;
1195     }
1196
1197     g_assert(buf.buffer != NULL);
1198     amanda_header = g_new(dumpfile_t, 1);
1199     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1200     pself->volume_header = amanda_header;
1201     g_free(buf.buffer);
1202
1203     if (amanda_header->type != F_TAPESTART) {
1204         device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
1205         return pself->status;
1206     }
1207
1208     pself->volume_label = g_strdup(amanda_header->name);
1209     pself->volume_time = g_strdup(amanda_header->datestamp);
1210     /* pself->volume_header is already set */
1211
1212     device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1213
1214     return pself->status;
1215 }
1216
1217 static gboolean
1218 s3_device_start (Device * pself, DeviceAccessMode mode,
1219                  char * label, char * timestamp) {
1220     S3Device * self;
1221
1222     self = S3_DEVICE(pself);
1223
1224     if (device_in_error(self)) return FALSE;
1225
1226     if (!setup_handle(self)) {
1227         /* setup_handle already set our error message */
1228         return FALSE;
1229     }
1230
1231     pself->access_mode = mode;
1232     pself->in_file = FALSE;
1233
1234     /* try creating the bucket, in case it doesn't exist */
1235     if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
1236         guint response_code;
1237         s3_error_code_t s3_error_code;
1238         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1239
1240         /* if it isn't an expected error (bucket already exists),
1241          * return FALSE */
1242         if (response_code != 409 ||
1243             s3_error_code != S3_ERROR_BucketAlreadyExists) {
1244             device_set_error(pself,
1245                 vstrallocf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3)),
1246                 DEVICE_STATUS_DEVICE_ERROR);
1247             return FALSE;
1248         }
1249     }
1250
1251     /* take care of any dirty work for this mode */
1252     switch (mode) {
1253         case ACCESS_READ:
1254             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1255                 /* s3_device_read_label already set our error message */
1256                 return FALSE;
1257             }
1258             break;
1259
1260         case ACCESS_WRITE:
1261             delete_all_files(self);
1262
1263             /* write a new amanda header */
1264             if (!write_amanda_header(self, label, timestamp)) {
1265                 return FALSE;
1266             }
1267
1268             pself->volume_label = newstralloc(pself->volume_label, label);
1269             pself->volume_time = newstralloc(pself->volume_time, timestamp);
1270
1271             /* unset the VOLUME_UNLABELED flag, if it was set */
1272             device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1273             break;
1274
1275         case ACCESS_APPEND:
1276             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1277                 /* s3_device_read_label already set our error message */
1278                 return FALSE;
1279             }
1280             return seek_to_end(self);
1281             break;
1282
1283         case ACCESS_NULL:
1284             g_assert_not_reached();
1285     }
1286
1287     return TRUE;
1288 }
1289
1290 static gboolean
1291 s3_device_finish (Device * pself) {
1292     if (device_in_error(pself)) return FALSE;
1293
1294     /* we're not in a file anymore */
1295     pself->access_mode = ACCESS_NULL;
1296
1297     return TRUE;
1298 }
1299
1300 /* functions for writing */
1301
1302
1303 static gboolean
1304 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
1305     S3Device *self = S3_DEVICE(pself);
1306     CurlBuffer amanda_header = {NULL, 0, 0, 0};
1307     gboolean result;
1308     size_t header_size;
1309     char *key;
1310
1311     if (device_in_error(self)) return FALSE;
1312
1313     pself->is_eom = FALSE;
1314
1315     /* Set the blocksize to zero, since there's no header to skip (it's stored
1316      * in a distinct file, rather than block zero) */
1317     jobInfo->blocksize = 0;
1318
1319     /* Build the amanda header. */
1320     header_size = 0; /* no minimum size */
1321     amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
1322         &header_size);
1323     if (amanda_header.buffer == NULL) {
1324         device_set_error(pself,
1325             stralloc(_("Amanda file header won't fit in a single block!")),
1326             DEVICE_STATUS_DEVICE_ERROR);
1327         return FALSE;
1328     }
1329     amanda_header.buffer_len = header_size;
1330
1331     /* set the file and block numbers correctly */
1332     pself->file = (pself->file > 0)? pself->file+1 : 1;
1333     pself->block = 0;
1334     pself->in_file = TRUE;
1335
1336     /* write it out as a special block (not the 0th) */
1337     key = special_file_to_key(self, "filestart", pself->file);
1338     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
1339                        &amanda_header, NULL, NULL);
1340     g_free(amanda_header.buffer);
1341     g_free(key);
1342     if (!result) {
1343         device_set_error(pself,
1344             vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3)),
1345             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1346         return FALSE;
1347     }
1348
1349     return TRUE;
1350 }
1351
1352 static gboolean
1353 s3_device_write_block (Device * pself, guint size, gpointer data) {
1354     gboolean result;
1355     char *filename;
1356     S3Device * self = S3_DEVICE(pself);
1357     CurlBuffer to_write = {data, size, 0, 0};
1358
1359     g_assert (self != NULL);
1360     g_assert (data != NULL);
1361     if (device_in_error(self)) return FALSE;
1362
1363     filename = file_and_block_to_key(self, pself->file, pself->block);
1364
1365     result = s3_upload(self->s3, self->bucket, filename, S3_BUFFER_READ_FUNCS,
1366         &to_write, NULL, NULL);
1367     g_free(filename);
1368     if (!result) {
1369         device_set_error(pself,
1370             vstrallocf(_("While writing data block to S3: %s"), s3_strerror(self->s3)),
1371             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1372         return FALSE;
1373     }
1374
1375     pself->block++;
1376
1377     return TRUE;
1378 }
1379
1380 static gboolean
1381 s3_device_finish_file (Device * pself) {
1382     if (device_in_error(pself)) return FALSE;
1383
1384     /* we're not in a file anymore */
1385     pself->in_file = FALSE;
1386
1387     return TRUE;
1388 }
1389
1390 static gboolean
1391 s3_device_recycle_file(Device *pself, guint file) {
1392     S3Device *self = S3_DEVICE(pself);
1393     if (device_in_error(self)) return FALSE;
1394
1395     return delete_file(self, file);
1396     /* delete_file already set our error message if necessary */
1397 }
1398
1399 static gboolean
1400 s3_device_erase(Device *pself) {
1401     S3Device *self = S3_DEVICE(pself);
1402     char *key = NULL;
1403     const char *errmsg = NULL;
1404     guint response_code;
1405     s3_error_code_t s3_error_code;
1406
1407     if (!setup_handle(self)) {
1408         /* error set by setup_handle */
1409         return FALSE;
1410     }
1411
1412     key = special_file_to_key(self, "tapestart", -1);
1413     if (!s3_delete(self->s3, self->bucket, key)) {
1414         s3_error(self->s3, &errmsg, NULL, NULL, NULL, NULL, NULL);
1415         device_set_error(pself,
1416             stralloc(errmsg),
1417             DEVICE_STATUS_DEVICE_ERROR);
1418         return FALSE;
1419     }
1420     g_free(key);
1421
1422     if (!delete_all_files(self))
1423         return FALSE;
1424
1425     if (!s3_delete_bucket(self->s3, self->bucket)) {
1426         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1427
1428         /*
1429          * ignore the error if the bucket isn't empty (there may be data from elsewhere)
1430          * or the bucket not existing (already deleted perhaps?)
1431          */
1432         if (!(
1433                 (response_code == 409 && s3_error_code == S3_ERROR_BucketNotEmpty) ||
1434                 (response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket))) {
1435
1436             device_set_error(pself,
1437                 stralloc(errmsg),
1438                 DEVICE_STATUS_DEVICE_ERROR);
1439             return FALSE;
1440         }
1441     }
1442     return TRUE;
1443 }
1444
1445 /* functions for reading */
1446
1447 static dumpfile_t*
1448 s3_device_seek_file(Device *pself, guint file) {
1449     S3Device *self = S3_DEVICE(pself);
1450     gboolean result;
1451     char *key;
1452     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1453     dumpfile_t *amanda_header;
1454     const char *errmsg = NULL;
1455
1456     if (device_in_error(self)) return NULL;
1457
1458     pself->file = file;
1459     pself->is_eof = FALSE;
1460     pself->in_file = FALSE;
1461     pself->block = 0;
1462
1463     /* read it in */
1464     key = special_file_to_key(self, "filestart", pself->file);
1465     result = s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
1466         &buf, NULL, NULL);
1467     g_free(key);
1468
1469     if (!result) {
1470         guint response_code;
1471         s3_error_code_t s3_error_code;
1472         s3_error(self->s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
1473
1474         /* if it's an expected error (not found), check what to do. */
1475         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1476             int next_file;
1477             next_file = find_next_file(self, pself->file);
1478             if (next_file > 0) {
1479                 /* Note short-circut of dispatcher. */
1480                 return s3_device_seek_file(pself, next_file);
1481             } else if (next_file == 0) {
1482                 /* No next file. Check if we are one past the end. */
1483                 key = special_file_to_key(self, "filestart", pself->file - 1);
1484                 result = s3_read(self->s3, self->bucket, key,
1485                     S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
1486                 g_free(key);
1487                 if (result) {
1488                     /* pself->file, etc. are already correct */
1489                     return make_tapeend_header();
1490                 } else {
1491                     device_set_error(pself,
1492                         stralloc(_("Attempt to read past tape-end file")),
1493                         DEVICE_STATUS_SUCCESS);
1494                     return NULL;
1495                 }
1496             }
1497         } else {
1498             /* An unexpected error occured finding out if we are the last file. */
1499             device_set_error(pself,
1500                 stralloc(errmsg),
1501                 DEVICE_STATUS_DEVICE_ERROR);
1502             return NULL;
1503         }
1504     }
1505
1506     /* and make a dumpfile_t out of it */
1507     g_assert(buf.buffer != NULL);
1508     amanda_header = g_new(dumpfile_t, 1);
1509     fh_init(amanda_header);
1510     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1511     g_free(buf.buffer);
1512
1513     switch (amanda_header->type) {
1514         case F_DUMPFILE:
1515         case F_CONT_DUMPFILE:
1516         case F_SPLIT_DUMPFILE:
1517             break;
1518
1519         default:
1520             device_set_error(pself,
1521                 stralloc(_("Invalid amanda header while reading file header")),
1522                 DEVICE_STATUS_VOLUME_ERROR);
1523             g_free(amanda_header);
1524             return NULL;
1525     }
1526
1527     pself->in_file = TRUE;
1528     return amanda_header;
1529 }
1530
1531 static gboolean
1532 s3_device_seek_block(Device *pself, guint64 block) {
1533     if (device_in_error(pself)) return FALSE;
1534
1535     pself->block = block;
1536     return TRUE;
1537 }
1538
1539 typedef struct s3_read_block_data {
1540     gpointer data;
1541     int size_req;
1542     int size_written;
1543
1544     CurlBuffer curl;
1545 } s3_read_block_data;
1546
1547 /* wrapper around s3_buffer_write_func to write as much data as possible to
1548  * the user's buffer, and switch to a dynamically allocated buffer if that
1549  * isn't large enough */
1550 static size_t
1551 s3_read_block_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
1552 {
1553     s3_read_block_data *dat = stream;
1554     guint new_bytes, bytes_needed;
1555
1556     /* if data is NULL, call through to s3_buffer_write_func */
1557     if (!dat->data) {
1558         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1559     }
1560
1561     new_bytes = (guint) size * nmemb;
1562     bytes_needed = dat->size_written + new_bytes;
1563
1564     if (bytes_needed > (guint)dat->size_req) {
1565         /* this read will overflow the user's buffer, so malloc ourselves
1566          * a new buffer and keep reading */
1567         dat->curl.buffer = g_malloc(bytes_needed);
1568         dat->curl.buffer_len = bytes_needed;
1569         dat->curl.buffer_pos = dat->size_written;
1570         memcpy(dat->curl.buffer, dat->data, dat->size_written);
1571         dat->data = NULL; /* signal that the user's buffer is too small */
1572         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1573     }
1574
1575     /* copy it into the dat->data buffer, and increment the size */
1576     memcpy(dat->data + dat->size_written, ptr, new_bytes);
1577     dat->size_written += new_bytes;
1578
1579     return new_bytes;
1580 }
1581
1582 static int
1583 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1584     S3Device * self = S3_DEVICE(pself);
1585     char *key;
1586     s3_read_block_data dat = {NULL, 0, 0, { NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE} };
1587     gboolean result;
1588
1589     g_assert (self != NULL);
1590     if (device_in_error(self)) return -1;
1591
1592     /* get the file*/
1593     key = file_and_block_to_key(self, pself->file, pself->block);
1594     g_assert(key != NULL);
1595     if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1596         if (*size_req >= self->cached_size) {
1597             /* use the cached copy and clear the cache */
1598             memcpy(data, self->cached_buf, self->cached_size);
1599             *size_req = self->cached_size;
1600
1601             g_free(key);
1602             g_free(self->cached_key);
1603             self->cached_key = NULL;
1604             g_free(self->cached_buf);
1605             self->cached_buf = NULL;
1606
1607             pself->block++;
1608             return *size_req;
1609         } else {
1610             *size_req = self->cached_size;
1611             g_free(key);
1612             return 0;
1613         }
1614     }
1615
1616     /* clear the cache, as it's useless to us */
1617     if (self->cached_key) {
1618         g_free(self->cached_key);
1619         self->cached_key = NULL;
1620
1621         g_free(self->cached_buf);
1622         self->cached_buf = NULL;
1623     }
1624
1625     /* set up dat for the write_func callback */
1626     if (!data || *size_req <= 0) {
1627         dat.data = NULL;
1628         dat.size_req = 0;
1629     } else {
1630         dat.data = data;
1631         dat.size_req = *size_req;
1632     }
1633
1634     result = s3_read(self->s3, self->bucket, key, s3_read_block_write_func,
1635         s3_buffer_reset_func, &dat, NULL, NULL);
1636     if (!result) {
1637         guint response_code;
1638         s3_error_code_t s3_error_code;
1639         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1640
1641         g_free(key);
1642         key = NULL;
1643
1644         /* if it's an expected error (not found), just return -1 */
1645         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1646             pself->is_eof = TRUE;
1647             pself->in_file = FALSE;
1648             device_set_error(pself,
1649                 stralloc(_("EOF")),
1650                 DEVICE_STATUS_SUCCESS);
1651             return -1;
1652         }
1653
1654         /* otherwise, log it and return FALSE */
1655         device_set_error(pself,
1656             vstrallocf(_("While reading data block from S3: %s"), s3_strerror(self->s3)),
1657             DEVICE_STATUS_VOLUME_ERROR);
1658         return -1;
1659     }
1660
1661     if (dat.data == NULL) {
1662         /* data was larger than the available space, so cache it and return
1663          * the actual size */
1664         self->cached_buf = dat.curl.buffer;
1665         self->cached_size = dat.curl.buffer_pos;
1666         self->cached_key = key;
1667         key = NULL;
1668
1669         *size_req = dat.curl.buffer_pos;
1670         return 0;
1671     }
1672
1673     /* ok, the read went directly to the user's buffer, so we need only
1674      * set and return the size */
1675     pself->block++;
1676     g_free(key);
1677     *size_req = dat.size_written;
1678     return dat.size_written;
1679 }