git.gag.com Git - debian/amanda/blob - device-src/s3-device.c

   1 /*
   2  * Copyright (c) 2005-2008 Zmanda Inc.  All Rights Reserved.
   3  *
   4  * This library is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU Lesser General Public License version 2.1 as
   6  * published by the Free Software Foundation.
   7  *
   8  * This library is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  10  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
  11  * License for more details.
  12  *
  13  * You should have received a copy of the GNU Lesser General Public License
  14  * along with this library; if not, write to the Free Software Foundation,
  15  * Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA.
  16  *
  17  * Contact information: Zmanda Inc., 465 S Mathlida Ave, Suite 300
  18  * Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
  19  */
  20
  21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
  22  * data.  It stores data in keys named with a user-specified prefix, inside a
  23  * user-specified bucket.  Data is stored in the form of numbered (large)
  24  * blocks.
  25  */
  26
  27 #include <string.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <unistd.h>
  31 #include <dirent.h>
  32 #include <regex.h>
  33 #include <time.h>
  34 #include "util.h"
  35 #include "amanda.h"
  36 #include "conffile.h"
  37 #include "device.h"
  38 #include "s3.h"
  39 #include <curl/curl.h>
  40 #ifdef HAVE_OPENSSL_HMAC_H
  41 # include <openssl/hmac.h>
  42 #else
  43 # ifdef HAVE_CRYPTO_HMAC_H
  44 #  include <crypto/hmac.h>
  45 # else
  46 #  ifdef HAVE_HMAC_H
  47 #   include <hmac.h>
  48 #  endif
  49 # endif
  50 #endif
  51
  52 /*
  53  * Type checking and casting macros
  54  */
  55 #define TYPE_S3_DEVICE  (s3_device_get_type())
  56 #define S3_DEVICE(obj)  G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
  57 #define S3_DEVICE_CONST(obj)    G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
  58 #define S3_DEVICE_CLASS(klass)  G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
  59 #define IS_S3_DEVICE(obj)       G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
  60
  61 #define S3_DEVICE_GET_CLASS(obj)        G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
  62 static GType    s3_device_get_type      (void);
  63
  64 /*
  65  * Main object structure
  66  */
  67 typedef struct _S3MetadataFile S3MetadataFile;
  68
  69 typedef struct _S3Device S3Device;
  70 struct _S3Device {
  71     Device __parent__;
  72
  73     /* The "easy" curl handle we use to access Amazon S3 */
  74     S3Handle *s3;
  75
  76     /* S3 access information */
  77     char *bucket;
  78     char *prefix;
  79
  80     /* The S3 access information. */
  81     char *secret_key;
  82     char *access_key;
  83     char *user_token;
  84     gboolean is_devpay;
  85
  86     char *bucket_location;
  87
  88     /* a cache for unsuccessful reads (where we get the file but the caller
  89      * doesn't have space for it or doesn't want it), where we expect the
  90      * next call will request the same file.
  91      */
  92     char *cached_buf;
  93     char *cached_key;
  94     int cached_size;
  95
  96     /* Produce verbose output? */
  97     gboolean verbose;
  98
  99     /* Use SSL? */
 100     gboolean use_ssl;
 101 };
 102
 103 /*
 104  * Class definition
 105  */
 106 typedef struct _S3DeviceClass S3DeviceClass;
 107 struct _S3DeviceClass {
 108     DeviceClass __parent__;
 109 };
 110
 111
 112 /*
 113  * Constants and static data
 114  */
 115
 116 #define S3_DEVICE_NAME "s3"
 117 #define DEVPAY_DEVICE_NAME "s3zmanda"
 118
 119 /* Maximum key length as specified in the S3 documentation
 120  * (*excluding* null terminator) */
 121 #define S3_MAX_KEY_LENGTH 1024
 122
 123 /* Note: for compatability, min can only be decreased and max increased */
 124 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
 125 #define S3_DEVICE_MAX_BLOCK_SIZE (100*1024*1024)
 126 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
 127
 128 /* This goes in lieu of file number for metadata. */
 129 #define SPECIAL_INFIX "special-"
 130
 131 /* pointer to the class of our parent */
 132 static DeviceClass *parent_class = NULL;
 133
 134 /*
 135  * device-specific properties
 136  */
 137
 138 /* Authentication information for Amazon S3. Both of these are strings. */
 139 static DevicePropertyBase device_property_s3_access_key;
 140 static DevicePropertyBase device_property_s3_secret_key;
 141 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
 142 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
 143
 144 /* Same, but for S3 with DevPay. */
 145 static DevicePropertyBase device_property_s3_user_token;
 146 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
 147
 148 /* Location constraint for new buckets created on Amazon S3. */
 149 static DevicePropertyBase device_property_s3_bucket_location;
 150 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
 151
 152 /* Whether to use SSL with Amazon S3. */
 153 static DevicePropertyBase device_property_s3_ssl;
 154 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
 155
 156
 157 /*
 158  * prototypes
 159  */
 160
 161 void s3_device_register(void);
 162
 163 /*
 164  * utility functions */
 165
 166 /* Given file and block numbers, return an S3 key.
 167  *
 168  * @param self: the S3Device object
 169  * @param file: the file number
 170  * @param block: the block within that file
 171  * @returns: a newly allocated string containing an S3 key.
 172  */
 173 static char *
 174 file_and_block_to_key(S3Device *self,
 175                       int file,
 176                       guint64 block);
 177
 178 /* Given the name of a special file (such as 'tapestart'), generate
 179  * the S3 key to use for that file.
 180  *
 181  * @param self: the S3Device object
 182  * @param special_name: name of the special file
 183  * @param file: a file number to include; omitted if -1
 184  * @returns: a newly alocated string containing an S3 key.
 185  */
 186 static char *
 187 special_file_to_key(S3Device *self,
 188                     char *special_name,
 189                     int file);
 190 /* Write an amanda header file to S3.
 191  *
 192  * @param self: the S3Device object
 193  * @param label: the volume label
 194  * @param timestamp: the volume timestamp
 195  */
 196 static gboolean
 197 write_amanda_header(S3Device *self,
 198                     char *label,
 199                     char * timestamp);
 200
 201 /* "Fast forward" this device to the end by looking up the largest file number
 202  * present and setting the current file number one greater.
 203  *
 204  * @param self: the S3Device object
 205  */
 206 static gboolean
 207 seek_to_end(S3Device *self);
 208
 209 /* Find the number of the last file that contains any data (even just a header).
 210  *
 211  * @param self: the S3Device object
 212  * @returns: the last file, or -1 in event of an error
 213  */
 214 static int
 215 find_last_file(S3Device *self);
 216
 217 /* Delete all blocks in the given file, including the filestart block
 218  *
 219  * @param self: the S3Device object
 220  * @param file: the file to delete
 221  */
 222 static gboolean
 223 delete_file(S3Device *self,
 224             int file);
 225
 226 /* Set up self->s3 as best as possible.
 227  *
 228  * The return value is TRUE iff self->s3 is useable.
 229  *
 230  * @param self: the S3Device object
 231  * @returns: TRUE if the handle is set up
 232  */
 233 static gboolean
 234 setup_handle(S3Device * self);
 235
 236 /*
 237  * class mechanics */
 238
 239 static void
 240 s3_device_init(S3Device * o);
 241
 242 static void
 243 s3_device_class_init(S3DeviceClass * c);
 244
 245 static void
 246 s3_device_finalize(GObject * o);
 247
 248 static Device*
 249 s3_device_factory(char * device_name, char * device_type, char * device_node);
 250
 251 /*
 252  * Property{Get,Set}Fns */
 253
 254 static gboolean s3_device_set_access_key_fn(Device *self,
 255     DevicePropertyBase *base, GValue *val,
 256     PropertySurety surety, PropertySource source);
 257
 258 static gboolean s3_device_set_secret_key_fn(Device *self,
 259     DevicePropertyBase *base, GValue *val,
 260     PropertySurety surety, PropertySource source);
 261
 262 static gboolean s3_device_set_user_token_fn(Device *self,
 263     DevicePropertyBase *base, GValue *val,
 264     PropertySurety surety, PropertySource source);
 265
 266 static gboolean s3_device_set_bucket_location_fn(Device *self,
 267     DevicePropertyBase *base, GValue *val,
 268     PropertySurety surety, PropertySource source);
 269
 270 static gboolean s3_device_set_verbose_fn(Device *self,
 271     DevicePropertyBase *base, GValue *val,
 272     PropertySurety surety, PropertySource source);
 273
 274 static gboolean s3_device_set_ssl_fn(Device *self,
 275     DevicePropertyBase *base, GValue *val,
 276     PropertySurety surety, PropertySource source);
 277
 278 /*
 279  * virtual functions */
 280
 281 static void
 282 s3_device_open_device(Device *pself, char *device_name,
 283                   char * device_type, char * device_node);
 284
 285 static DeviceStatusFlags s3_device_read_label(Device * self);
 286
 287 static gboolean
 288 s3_device_start(Device * self,
 289                 DeviceAccessMode mode,
 290                 char * label,
 291                 char * timestamp);
 292
 293 static gboolean
 294 s3_device_finish(Device * self);
 295
 296 static gboolean
 297 s3_device_start_file(Device * self,
 298                      dumpfile_t * jobInfo);
 299
 300 static gboolean
 301 s3_device_write_block(Device * self,
 302                       guint size,
 303                       gpointer data);
 304
 305 static gboolean
 306 s3_device_finish_file(Device * self);
 307
 308 static dumpfile_t*
 309 s3_device_seek_file(Device *pself,
 310                     guint file);
 311
 312 static gboolean
 313 s3_device_seek_block(Device *pself,
 314                      guint64 block);
 315
 316 static int
 317 s3_device_read_block(Device * pself,
 318                      gpointer data,
 319                      int *size_req);
 320
 321 static gboolean
 322 s3_device_recycle_file(Device *pself,
 323                        guint file);
 324
 325 /*
 326  * Private functions
 327  */
 328
 329 static char *
 330 file_and_block_to_key(S3Device *self,
 331                       int file,
 332                       guint64 block)
 333 {
 334     char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
 335                                    self->prefix, file, (long long unsigned int)block);
 336     g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
 337     return s3_key;
 338 }
 339
 340 static char *
 341 special_file_to_key(S3Device *self,
 342                     char *special_name,
 343                     int file)
 344 {
 345     if (file == -1)
 346         return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
 347     else
 348         return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
 349 }
 350
 351 static gboolean
 352 write_amanda_header(S3Device *self,
 353                     char *label,
 354                     char * timestamp)
 355 {
 356     CurlBuffer amanda_header = {NULL, 0, 0, 0};
 357     char * key = NULL;
 358     gboolean header_fits, result;
 359     dumpfile_t * dumpinfo = NULL;
 360     Device *d_self = DEVICE(self);
 361
 362     /* build the header */
 363     dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
 364     amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
 365         /* casting guint* to int* */
 366         (int*) &amanda_header.buffer_len, &header_fits);
 367     if (!header_fits) {
 368         device_set_error(d_self,
 369             stralloc(_("Amanda tapestart header won't fit in a single block!")),
 370             DEVICE_STATUS_DEVICE_ERROR);
 371         g_free(amanda_header.buffer);
 372         return FALSE;
 373     }
 374
 375     /* write out the header and flush the uploads. */
 376     key = special_file_to_key(self, "tapestart", -1);
 377     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
 378                        &amanda_header, NULL, NULL);
 379     g_free(amanda_header.buffer);
 380     g_free(key);
 381
 382     if (!result) {
 383         device_set_error(d_self,
 384             vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3)),
 385             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 386     }
 387     return result;
 388 }
 389
 390 static gboolean
 391 seek_to_end(S3Device *self) {
 392     int last_file;
 393
 394     Device *pself = DEVICE(self);
 395
 396     last_file = find_last_file(self);
 397     if (last_file < 0)
 398         return FALSE;
 399
 400     pself->file = last_file;
 401
 402     return TRUE;
 403 }
 404
 405 /* Convert an object name into a file number, assuming the given prefix
 406  * length. Returns -1 if the object name is invalid, or 0 if the object name
 407  * is a "special" key. */
 408 static int key_to_file(guint prefix_len, const char * key) {
 409     int file;
 410     int i;
 411
 412     /* skip the prefix */
 413     if (strlen(key) <= prefix_len)
 414         return -1;
 415
 416     key += prefix_len;
 417
 418     if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
 419         return 0;
 420     }
 421
 422     /* check that key starts with 'f' */
 423     if (key[0] != 'f')
 424         return -1;
 425     key++;
 426
 427     /* check that key is of the form "%08x-" */
 428     for (i = 0; i < 8; i++) {
 429         if (!(key[i] >= '0' && key[i] <= '9') &&
 430             !(key[i] >= 'a' && key[i] <= 'f') &&
 431             !(key[i] >= 'A' && key[i] <= 'F')) break;
 432     }
 433     if (key[i] != '-') return -1;
 434     if (i < 8) return -1;
 435
 436     /* convert the file number */
 437     errno = 0;
 438     file = strtoul(key, NULL, 16);
 439     if (errno != 0) {
 440         g_warning(_("unparseable file number '%s'"), key);
 441         return -1;
 442     }
 443
 444     return file;
 445 }
 446
 447 /* Find the number of the last file that contains any data (even just a header).
 448  * Returns -1 in event of an error
 449  */
 450 static int
 451 find_last_file(S3Device *self) {
 452     gboolean result;
 453     GSList *keys;
 454     unsigned int prefix_len = strlen(self->prefix);
 455     int last_file = 0;
 456     Device *d_self = DEVICE(self);
 457
 458     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 459     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 460     if (!result) {
 461         device_set_error(d_self,
 462             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 463             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 464         return -1;
 465     }
 466
 467     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 468         int file = key_to_file(prefix_len, keys->data);
 469
 470         /* and if it's the last, keep it */
 471         if (file > last_file)
 472             last_file = file;
 473     }
 474
 475     return last_file;
 476 }
 477
 478 /* Find the number of the file following the requested one, if any.
 479  * Returns 0 if there is no such file or -1 in event of an error
 480  */
 481 static int
 482 find_next_file(S3Device *self, int last_file) {
 483     gboolean result;
 484     GSList *keys;
 485     unsigned int prefix_len = strlen(self->prefix);
 486     int next_file = 0;
 487     Device *d_self = DEVICE(self);
 488
 489     /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
 490     result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
 491     if (!result) {
 492         device_set_error(d_self,
 493             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 494             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 495         return -1;
 496     }
 497
 498     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 499         int file;
 500
 501         file = key_to_file(prefix_len, (char*)keys->data);
 502
 503         if (file < 0) {
 504             /* Set this in case we don't find a next file; this is not a
 505              * hard error, so if we can find a next file we'll return that
 506              * instead. */
 507             next_file = -1;
 508         }
 509
 510         if (file < next_file && file > last_file) {
 511             next_file = file;
 512         }
 513     }
 514
 515     return last_file;
 516 }
 517
 518 static gboolean
 519 delete_file(S3Device *self,
 520             int file)
 521 {
 522     gboolean result;
 523     GSList *keys;
 524     char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
 525     Device *d_self = DEVICE(self);
 526
 527     result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
 528     if (!result) {
 529         device_set_error(d_self,
 530             vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
 531             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
 532         return FALSE;
 533     }
 534
 535     /* this will likely be a *lot* of keys */
 536     for (; keys; keys = g_slist_remove(keys, keys->data)) {
 537         if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
 538         if (!s3_delete(self->s3, self->bucket, keys->data)) {
 539             device_set_error(d_self,
 540                 vstrallocf(_("While deleting key '%s': %s"),
 541                             (char*)keys->data, s3_strerror(self->s3)),
 542                 DEVICE_STATUS_DEVICE_ERROR);
 543             g_slist_free(keys);
 544             return FALSE;
 545         }
 546     }
 547
 548     return TRUE;
 549 }
 550
 551 /*
 552  * Class mechanics
 553  */
 554
 555 void
 556 s3_device_register(void)
 557 {
 558     static const char * device_prefix_list[] = { S3_DEVICE_NAME, DEVPAY_DEVICE_NAME, NULL };
 559     g_assert(s3_init());
 560
 561     /* set up our properties */
 562     device_property_fill_and_register(&device_property_s3_secret_key,
 563                                       G_TYPE_STRING, "s3_secret_key",
 564        "Secret access key to authenticate with Amazon S3");
 565     device_property_fill_and_register(&device_property_s3_access_key,
 566                                       G_TYPE_STRING, "s3_access_key",
 567        "Access key ID to authenticate with Amazon S3");
 568     device_property_fill_and_register(&device_property_s3_user_token,
 569                                       G_TYPE_STRING, "s3_user_token",
 570        "User token for authentication Amazon devpay requests");
 571     device_property_fill_and_register(&device_property_s3_bucket_location,
 572                                       G_TYPE_STRING, "s3_bucket_location",
 573        "Location constraint for buckets on Amazon S3");
 574     device_property_fill_and_register(&device_property_s3_ssl,
 575                                       G_TYPE_BOOLEAN, "s3_ssl",
 576        "Whether to use SSL with Amazon S3");
 577
 578
 579     /* register the device itself */
 580     register_device(s3_device_factory, device_prefix_list);
 581 }
 582
 583 static GType
 584 s3_device_get_type(void)
 585 {
 586     static GType type = 0;
 587
 588     if G_UNLIKELY(type == 0) {
 589         static const GTypeInfo info = {
 590             sizeof (S3DeviceClass),
 591             (GBaseInitFunc) NULL,
 592             (GBaseFinalizeFunc) NULL,
 593             (GClassInitFunc) s3_device_class_init,
 594             (GClassFinalizeFunc) NULL,
 595             NULL /* class_data */,
 596             sizeof (S3Device),
 597             0 /* n_preallocs */,
 598             (GInstanceInitFunc) s3_device_init,
 599             NULL
 600         };
 601
 602         type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
 603                                        (GTypeFlags)0);
 604     }
 605
 606     return type;
 607 }
 608
 609 static void
 610 s3_device_init(S3Device * self)
 611 {
 612     Device * dself = DEVICE(self);
 613     GValue response;
 614
 615     /* Register property values
 616      * Note: Some aren't added until s3_device_open_device()
 617      */
 618     bzero(&response, sizeof(response));
 619
 620     g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
 621     g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
 622     device_set_simple_property(dself, PROPERTY_CONCURRENCY,
 623             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 624     g_value_unset(&response);
 625
 626     g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
 627     g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
 628     device_set_simple_property(dself, PROPERTY_STREAMING,
 629             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 630     g_value_unset(&response);
 631
 632     g_value_init(&response, G_TYPE_BOOLEAN);
 633     g_value_set_boolean(&response, TRUE);
 634     device_set_simple_property(dself, PROPERTY_APPENDABLE,
 635             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 636     g_value_unset(&response);
 637
 638     g_value_init(&response, G_TYPE_BOOLEAN);
 639     g_value_set_boolean(&response, TRUE);
 640     device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
 641             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 642     g_value_unset(&response);
 643
 644     g_value_init(&response, G_TYPE_BOOLEAN);
 645     g_value_set_boolean(&response, FALSE);
 646     device_set_simple_property(dself, PROPERTY_COMPRESSION,
 647             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 648     g_value_unset(&response);
 649
 650     g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
 651     g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
 652     device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
 653             &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
 654     g_value_unset(&response);
 655
 656 }
 657
 658 static void
 659 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
 660 {
 661     GObjectClass *g_object_class = (GObjectClass*) c;
 662     DeviceClass *device_class = (DeviceClass *)c;
 663
 664     parent_class = g_type_class_ref (TYPE_DEVICE);
 665
 666     device_class->open_device = s3_device_open_device;
 667     device_class->read_label = s3_device_read_label;
 668     device_class->start = s3_device_start;
 669     device_class->finish = s3_device_finish;
 670
 671     device_class->start_file = s3_device_start_file;
 672     device_class->write_block = s3_device_write_block;
 673     device_class->finish_file = s3_device_finish_file;
 674
 675     device_class->seek_file = s3_device_seek_file;
 676     device_class->seek_block = s3_device_seek_block;
 677     device_class->read_block = s3_device_read_block;
 678     device_class->recycle_file = s3_device_recycle_file;
 679
 680     g_object_class->finalize = s3_device_finalize;
 681
 682     device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
 683             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 684             device_simple_property_get_fn,
 685             s3_device_set_access_key_fn);
 686
 687     device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
 688             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 689             device_simple_property_get_fn,
 690             s3_device_set_secret_key_fn);
 691
 692     device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
 693             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 694             device_simple_property_get_fn,
 695             s3_device_set_user_token_fn);
 696
 697     device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
 698             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 699             device_simple_property_get_fn,
 700             s3_device_set_bucket_location_fn);
 701
 702     device_class_register_property(device_class, PROPERTY_VERBOSE,
 703             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 704             device_simple_property_get_fn,
 705             s3_device_set_verbose_fn);
 706
 707     device_class_register_property(device_class, PROPERTY_S3_SSL,
 708             PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
 709             device_simple_property_get_fn,
 710             s3_device_set_ssl_fn);
 711
 712     device_class_register_property(device_class, PROPERTY_COMPRESSION,
 713             PROPERTY_ACCESS_GET_MASK,
 714             device_simple_property_get_fn,
 715             NULL);
 716 }
 717
 718 static gboolean
 719 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
 720     GValue *val, PropertySurety surety, PropertySource source)
 721 {
 722     S3Device *self = S3_DEVICE(p_self);
 723
 724     amfree(self->access_key);
 725     self->access_key = g_value_dup_string(val);
 726     device_clear_volume_details(p_self);
 727
 728     return device_simple_property_set_fn(p_self, base, val, surety, source);
 729 }
 730
 731 static gboolean
 732 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
 733     GValue *val, PropertySurety surety, PropertySource source)
 734 {
 735     S3Device *self = S3_DEVICE(p_self);
 736
 737     amfree(self->secret_key);
 738     self->secret_key = g_value_dup_string(val);
 739     device_clear_volume_details(p_self);
 740
 741     return device_simple_property_set_fn(p_self, base, val, surety, source);
 742 }
 743
 744 static gboolean
 745 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
 746     GValue *val, PropertySurety surety, PropertySource source)
 747 {
 748     S3Device *self = S3_DEVICE(p_self);
 749
 750     if (!self->is_devpay) {
 751         device_set_error(p_self, stralloc(_(
 752                    "Can't set a user token unless DevPay is in use")),
 753             DEVICE_STATUS_DEVICE_ERROR);
 754         return FALSE;
 755     }
 756
 757     amfree(self->user_token);
 758     self->user_token = g_value_dup_string(val);
 759     device_clear_volume_details(p_self);
 760
 761     return device_simple_property_set_fn(p_self, base, val, surety, source);
 762 }
 763
 764 static gboolean
 765 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
 766     GValue *val, PropertySurety surety, PropertySource source)
 767 {
 768     S3Device *self = S3_DEVICE(p_self);
 769
 770     if (self->use_ssl && !s3_curl_location_compat()) {
 771         device_set_error(p_self, stralloc(_(
 772                 "Location constraint given for Amazon S3 bucket, "
 773                 "but libcurl is too old support wildcard certificates.")),
 774             DEVICE_STATUS_DEVICE_ERROR);
 775        return FALSE;
 776     }
 777
 778     if (!s3_bucket_location_compat(self->bucket)) {
 779         device_set_error(p_self, g_strdup_printf(_(
 780                 "Location constraint given for Amazon S3 bucket, "
 781                 "but the bucket name (%s) is not usable as a subdomain."),
 782                 self->bucket),
 783             DEVICE_STATUS_DEVICE_ERROR);
 784        return FALSE;
 785     }
 786
 787     amfree(self->bucket_location);
 788     self->bucket_location = g_value_dup_string(val);
 789     device_clear_volume_details(p_self);
 790
 791     return device_simple_property_set_fn(p_self, base, val, surety, source);
 792 }
 793
 794 static gboolean
 795 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
 796     GValue *val, PropertySurety surety, PropertySource source)
 797 {
 798     S3Device *self = S3_DEVICE(p_self);
 799
 800     self->verbose = g_value_get_boolean(val);
 801     /* Our S3 handle may not yet have been instantiated; if so, it will
 802      * get the proper verbose setting when it is created */
 803     if (self->s3)
 804         s3_verbose(self->s3, self->verbose);
 805
 806     return device_simple_property_set_fn(p_self, base, val, surety, source);
 807 }
 808
 809 static gboolean
 810 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
 811     GValue *val, PropertySurety surety, PropertySource source)
 812 {
 813     S3Device *self = S3_DEVICE(p_self);
 814     gboolean new_val;
 815
 816     new_val = g_value_get_boolean(val);
 817     /* Our S3 handle may not yet have been instantiated; if so, it will
 818      * get the proper use_ssl setting when it is created */
 819     if (self->s3 && !s3_use_ssl(self->s3, new_val)) {
 820         device_set_error(p_self, g_strdup_printf(_(
 821                 "Error setting S3 SSL/TLS use "
 822                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
 823             DEVICE_STATUS_DEVICE_ERROR);
 824         return FALSE;
 825     }
 826     self->use_ssl = new_val;
 827
 828     return device_simple_property_set_fn(p_self, base, val, surety, source);
 829 }
 830
 831 static Device*
 832 s3_device_factory(char * device_name, char * device_type, char * device_node)
 833 {
 834     Device *rval;
 835     S3Device * s3_rval;
 836     g_assert(0 == strcmp(device_type, S3_DEVICE_NAME) ||
 837              0 == strcmp(device_type, DEVPAY_DEVICE_NAME));
 838     rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
 839     s3_rval = (S3Device*)rval;
 840
 841     device_open_device(rval, device_name, device_type, device_node);
 842     return rval;
 843 }
 844
 845 /*
 846  * Virtual function overrides
 847  */
 848
 849 static void
 850 s3_device_open_device(Device *pself, char *device_name,
 851                         char * device_type, char * device_node)
 852 {
 853     S3Device *self = S3_DEVICE(pself);
 854     char * name_colon;
 855     GValue tmp_value;
 856
 857     pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
 858     pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
 859     pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
 860
 861     /* Device name may be bucket/prefix, to support multiple volumes in a
 862      * single bucket. */
 863     name_colon = strchr(device_node, '/');
 864     if (name_colon == NULL) {
 865         self->bucket = g_strdup(device_node);
 866         self->prefix = g_strdup("");
 867     } else {
 868         self->bucket = g_strndup(device_node, name_colon - device_node);
 869         self->prefix = g_strdup(name_colon + 1);
 870     }
 871
 872     self->is_devpay = !strcmp(device_type, DEVPAY_DEVICE_NAME);
 873
 874     if (self->bucket == NULL || self->bucket[0] == '\0') {
 875         device_set_error(pself,
 876             vstrallocf(_("Empty bucket name in device %s"), device_name),
 877             DEVICE_STATUS_DEVICE_ERROR);
 878         amfree(self->bucket);
 879         amfree(self->prefix);
 880         return;
 881     }
 882
 883     g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
 884
 885     /* default values */
 886     self->verbose = FALSE;
 887
 888     /* use SSL if available */
 889     self->use_ssl = s3_curl_supports_ssl();
 890     bzero(&tmp_value, sizeof(GValue));
 891     g_value_init(&tmp_value, G_TYPE_BOOLEAN);
 892     g_value_set_boolean(&tmp_value, self->use_ssl);
 893     device_set_simple_property(pself, device_property_s3_ssl.ID,
 894         &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
 895
 896     if (parent_class->open_device) {
 897         parent_class->open_device(pself, device_name, device_type, device_node);
 898     }
 899 }
 900
 901 static void s3_device_finalize(GObject * obj_self) {
 902     S3Device *self = S3_DEVICE (obj_self);
 903
 904     if(G_OBJECT_CLASS(parent_class)->finalize)
 905         (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
 906
 907     if(self->s3) s3_free(self->s3);
 908     if(self->bucket) g_free(self->bucket);
 909     if(self->prefix) g_free(self->prefix);
 910     if(self->access_key) g_free(self->access_key);
 911     if(self->secret_key) g_free(self->secret_key);
 912     if(self->user_token) g_free(self->user_token);
 913     if(self->bucket_location) g_free(self->bucket_location);
 914 }
 915
 916 static gboolean setup_handle(S3Device * self) {
 917     Device *d_self = DEVICE(self);
 918     if (self->s3 == NULL) {
 919
 920         if (self->access_key == NULL || self->access_key[0] == '\0') {
 921             device_set_error(d_self,
 922                 stralloc(_("No Amazon access key specified")),
 923                 DEVICE_STATUS_DEVICE_ERROR);
 924             return FALSE;
 925         }
 926
 927         if (self->secret_key == NULL || self->secret_key[0] == '\0') {
 928             device_set_error(d_self,
 929                 stralloc(_("No Amazon secret key specified")),
 930                 DEVICE_STATUS_DEVICE_ERROR);
 931             return FALSE;
 932         }
 933
 934         if (self->is_devpay && self->user_token == NULL) {
 935             device_set_error(d_self,
 936                 stralloc(_("No Amazon user token specified")),
 937                 DEVICE_STATUS_DEVICE_ERROR);
 938             return FALSE;
 939         }
 940
 941         self->s3 = s3_open(self->access_key, self->secret_key, self->user_token,
 942             self->bucket_location);
 943         if (self->s3 == NULL) {
 944             device_set_error(d_self,
 945                 stralloc(_("Internal error creating S3 handle")),
 946                 DEVICE_STATUS_DEVICE_ERROR);
 947             return FALSE;
 948         }
 949     }
 950
 951     s3_verbose(self->s3, self->verbose);
 952
 953     if (!s3_use_ssl(self->s3, self->use_ssl)) {
 954         device_set_error(d_self, g_strdup_printf(_(
 955                 "Error setting S3 SSL/TLS use "
 956                 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
 957             DEVICE_STATUS_DEVICE_ERROR);
 958         return FALSE;
 959     }
 960
 961     return TRUE;
 962 }
 963
 964 static DeviceStatusFlags
 965 s3_device_read_label(Device *pself) {
 966     S3Device *self = S3_DEVICE(pself);
 967     char *key;
 968     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
 969     dumpfile_t *amanda_header;
 970
 971     /* note that this may be called from s3_device_start, when
 972      * self->access_mode is not ACCESS_NULL */
 973
 974     amfree(pself->volume_label);
 975     amfree(pself->volume_time);
 976     amfree(pself->volume_header);
 977
 978     if (device_in_error(self)) return pself->status;
 979
 980     if (!setup_handle(self)) {
 981         /* setup_handle already set our error message */
 982         return pself->status;
 983     }
 984
 985     key = special_file_to_key(self, "tapestart", -1);
 986     if (!s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
 987         guint response_code;
 988         s3_error_code_t s3_error_code;
 989         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
 990
 991         /* if it's an expected error (not found), just return FALSE */
 992         if (response_code == 404 &&
 993              (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
 994             g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
 995             device_set_error(pself,
 996                 stralloc(_("Amanda header not found -- unlabeled volume?")),
 997                   DEVICE_STATUS_DEVICE_ERROR
 998                 | DEVICE_STATUS_VOLUME_ERROR
 999                 | DEVICE_STATUS_VOLUME_UNLABELED);
1000             return pself->status;
1001         }
1002
1003         /* otherwise, log it and return */
1004         device_set_error(pself,
1005             vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3)),
1006             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1007         return pself->status;
1008     }
1009
1010     g_assert(buf.buffer != NULL);
1011     amanda_header = g_new(dumpfile_t, 1);
1012     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1013     pself->volume_header = amanda_header;
1014
1015     g_free(buf.buffer);
1016
1017     if (amanda_header->type != F_TAPESTART) {
1018         device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
1019         return pself->status;
1020     }
1021
1022     pself->volume_label = g_strdup(amanda_header->name);
1023     pself->volume_time = g_strdup(amanda_header->datestamp);
1024     /* pself->volume_header is already set */
1025
1026     device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1027
1028     return pself->status;
1029 }
1030
1031 static gboolean
1032 s3_device_start (Device * pself, DeviceAccessMode mode,
1033                  char * label, char * timestamp) {
1034     S3Device * self;
1035     int file, last_file;
1036
1037     self = S3_DEVICE(pself);
1038
1039     if (device_in_error(self)) return FALSE;
1040
1041     if (!setup_handle(self)) {
1042         /* setup_handle already set our error message */
1043         return FALSE;
1044     }
1045
1046     pself->access_mode = mode;
1047     pself->in_file = FALSE;
1048
1049     /* try creating the bucket, in case it doesn't exist */
1050     if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
1051         guint response_code;
1052         s3_error_code_t s3_error_code;
1053         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1054
1055         /* if it isn't an expected error (bucket already exists),
1056          * return FALSE */
1057         if (response_code != 409 ||
1058             s3_error_code != S3_ERROR_BucketAlreadyExists) {
1059             device_set_error(pself,
1060                 vstrallocf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3)),
1061                 DEVICE_STATUS_DEVICE_ERROR);
1062             return FALSE;
1063         }
1064     }
1065
1066     /* take care of any dirty work for this mode */
1067     switch (mode) {
1068         case ACCESS_READ:
1069             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1070                 /* s3_device_read_label already set our error message */
1071                 return FALSE;
1072             }
1073             break;
1074
1075         case ACCESS_WRITE:
1076             /* delete all files */
1077             last_file = find_last_file(self);
1078             if (last_file < 0) return FALSE;
1079             for (file = 0; file <= last_file; file++) {
1080                 if (!delete_file(self, file))
1081                     /* delete_file already set our error message */
1082                     return FALSE;
1083             }
1084
1085             /* write a new amanda header */
1086             if (!write_amanda_header(self, label, timestamp)) {
1087                 return FALSE;
1088             }
1089
1090             pself->volume_label = newstralloc(pself->volume_label, label);
1091             pself->volume_time = newstralloc(pself->volume_time, timestamp);
1092
1093             /* unset the VOLUME_UNLABELED flag, if it was set */
1094             device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1095             break;
1096
1097         case ACCESS_APPEND:
1098             if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1099                 /* s3_device_read_label already set our error message */
1100                 return FALSE;
1101             }
1102             return seek_to_end(self);
1103             break;
1104
1105         case ACCESS_NULL:
1106             g_assert_not_reached();
1107     }
1108
1109     return TRUE;
1110 }
1111
1112 static gboolean
1113 s3_device_finish (Device * pself) {
1114     if (device_in_error(pself)) return FALSE;
1115
1116     /* we're not in a file anymore */
1117     pself->access_mode = ACCESS_NULL;
1118
1119     return TRUE;
1120 }
1121
1122 /* functions for writing */
1123
1124
1125 static gboolean
1126 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
1127     S3Device *self = S3_DEVICE(pself);
1128     CurlBuffer amanda_header = {NULL, 0, 0, 0};
1129     gboolean header_fits, result;
1130     char *key;
1131
1132     if (device_in_error(self)) return FALSE;
1133
1134     /* Set the blocksize to zero, since there's no header to skip (it's stored
1135      * in a distinct file, rather than block zero) */
1136     jobInfo->blocksize = 0;
1137
1138     /* Build the amanda header. */
1139     amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
1140         (int *) &amanda_header.buffer_len, &header_fits);
1141     if (!header_fits) {
1142         device_set_error(pself,
1143             stralloc(_("Amanda file header won't fit in a single block!")),
1144             DEVICE_STATUS_DEVICE_ERROR);
1145         return FALSE;
1146     }
1147
1148     /* set the file and block numbers correctly */
1149     pself->file = (pself->file > 0)? pself->file+1 : 1;
1150     pself->block = 0;
1151     pself->in_file = TRUE;
1152
1153     /* write it out as a special block (not the 0th) */
1154     key = special_file_to_key(self, "filestart", pself->file);
1155     result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
1156                        &amanda_header, NULL, NULL);
1157     g_free(amanda_header.buffer);
1158     g_free(key);
1159     if (!result) {
1160         device_set_error(pself,
1161             vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3)),
1162             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1163         return FALSE;
1164     }
1165
1166     return TRUE;
1167 }
1168
1169 static gboolean
1170 s3_device_write_block (Device * pself, guint size, gpointer data) {
1171     gboolean result;
1172     char *filename;
1173     S3Device * self = S3_DEVICE(pself);
1174     CurlBuffer to_write = {data, size, 0, 0};
1175
1176     g_assert (self != NULL);
1177     g_assert (data != NULL);
1178     if (device_in_error(self)) return FALSE;
1179
1180     filename = file_and_block_to_key(self, pself->file, pself->block);
1181
1182     result = s3_upload(self->s3, self->bucket, filename, S3_BUFFER_READ_FUNCS,
1183         &to_write, NULL, NULL);
1184     g_free(filename);
1185     if (!result) {
1186         device_set_error(pself,
1187             vstrallocf(_("While writing data block to S3: %s"), s3_strerror(self->s3)),
1188             DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1189         return FALSE;
1190     }
1191
1192     pself->block++;
1193
1194     return TRUE;
1195 }
1196
1197 static gboolean
1198 s3_device_finish_file (Device * pself) {
1199     if (device_in_error(pself)) return FALSE;
1200
1201     /* we're not in a file anymore */
1202     pself->in_file = FALSE;
1203
1204     return TRUE;
1205 }
1206
1207 static gboolean
1208 s3_device_recycle_file(Device *pself, guint file) {
1209     S3Device *self = S3_DEVICE(pself);
1210     if (device_in_error(self)) return FALSE;
1211
1212     return delete_file(self, file);
1213     /* delete_file already set our error message if necessary */
1214 }
1215
1216 /* functions for reading */
1217
1218 static dumpfile_t*
1219 s3_device_seek_file(Device *pself, guint file) {
1220     S3Device *self = S3_DEVICE(pself);
1221     gboolean result;
1222     char *key;
1223     CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1224     dumpfile_t *amanda_header;
1225     const char *errmsg = NULL;
1226
1227     if (device_in_error(self)) return NULL;
1228
1229     pself->file = file;
1230     pself->is_eof = FALSE;
1231     pself->in_file = FALSE;
1232     pself->block = 0;
1233
1234     /* read it in */
1235     key = special_file_to_key(self, "filestart", pself->file);
1236     result = s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
1237         &buf, NULL, NULL);
1238     g_free(key);
1239
1240     if (!result) {
1241         guint response_code;
1242         s3_error_code_t s3_error_code;
1243         s3_error(self->s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
1244
1245         /* if it's an expected error (not found), check what to do. */
1246         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1247             int next_file;
1248             next_file = find_next_file(self, pself->file);
1249             if (next_file > 0) {
1250                 /* Note short-circut of dispatcher. */
1251                 return s3_device_seek_file(pself, next_file);
1252             } else if (next_file == 0) {
1253                 /* No next file. Check if we are one past the end. */
1254                 key = special_file_to_key(self, "filestart", pself->file - 1);
1255                 result = s3_read(self->s3, self->bucket, key,
1256                     S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
1257                 g_free(key);
1258                 if (result) {
1259                     /* pself->file, etc. are already correct */
1260                     return make_tapeend_header();
1261                 } else {
1262                     device_set_error(pself,
1263                         stralloc(_("Attempt to read past tape-end file")),
1264                         DEVICE_STATUS_SUCCESS);
1265                     return NULL;
1266                 }
1267             }
1268         } else {
1269             /* An unexpected error occured finding out if we are the last file. */
1270             device_set_error(pself,
1271                 stralloc(errmsg),
1272                 DEVICE_STATUS_DEVICE_ERROR);
1273             return NULL;
1274         }
1275     }
1276
1277     /* and make a dumpfile_t out of it */
1278     g_assert(buf.buffer != NULL);
1279     amanda_header = g_new(dumpfile_t, 1);
1280     fh_init(amanda_header);
1281     parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1282     g_free(buf.buffer);
1283
1284     switch (amanda_header->type) {
1285         case F_DUMPFILE:
1286         case F_CONT_DUMPFILE:
1287         case F_SPLIT_DUMPFILE:
1288             break;
1289
1290         default:
1291             device_set_error(pself,
1292                 stralloc(_("Invalid amanda header while reading file header")),
1293                 DEVICE_STATUS_VOLUME_ERROR);
1294             g_free(amanda_header);
1295             return NULL;
1296     }
1297
1298     pself->in_file = TRUE;
1299     return amanda_header;
1300 }
1301
1302 static gboolean
1303 s3_device_seek_block(Device *pself, guint64 block) {
1304     if (device_in_error(pself)) return FALSE;
1305
1306     pself->block = block;
1307     return TRUE;
1308 }
1309
1310 typedef struct s3_read_block_data {
1311     gpointer data;
1312     int size_req;
1313     int size_written;
1314
1315     CurlBuffer curl;
1316 } s3_read_block_data;
1317
1318 /* wrapper around s3_buffer_write_func to write as much data as possible to
1319  * the user's buffer, and switch to a dynamically allocated buffer if that
1320  * isn't large enough */
1321 static size_t
1322 s3_read_block_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
1323 {
1324     s3_read_block_data *dat = stream;
1325     guint new_bytes, bytes_needed;
1326
1327     /* if data is NULL, call through to s3_buffer_write_func */
1328     if (!dat->data) {
1329         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1330     }
1331
1332     new_bytes = (guint) size * nmemb;
1333     bytes_needed = dat->size_written + new_bytes;
1334
1335     if (bytes_needed > (guint)dat->size_written) {
1336         /* this read will overflow the user's buffer, so malloc ourselves
1337          * a new buffer and keep reading */
1338         dat->curl.buffer = g_malloc(bytes_needed);
1339         dat->curl.buffer_len = bytes_needed;
1340         dat->curl.buffer_pos = dat->size_written;
1341         memcpy(dat->curl.buffer, dat->data, dat->size_written);
1342         dat->data = NULL; /* signal that the user's buffer is too small */
1343         return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1344     }
1345
1346     memcpy(dat->data + dat->size_written, ptr, bytes_needed);
1347     return new_bytes;
1348 }
1349
1350 static int
1351 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1352     S3Device * self = S3_DEVICE(pself);
1353     char *key;
1354     s3_read_block_data dat = {NULL, 0, 0, { NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE} };
1355     gboolean result;
1356
1357     g_assert (self != NULL);
1358     if (device_in_error(self)) return -1;
1359
1360     /* get the file*/
1361     key = file_and_block_to_key(self, pself->file, pself->block);
1362     g_assert(key != NULL);
1363     if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1364         if (*size_req >= self->cached_size) {
1365             /* use the cached copy and clear the cache */
1366             memcpy(data, self->cached_buf, self->cached_size);
1367             *size_req = self->cached_size;
1368
1369             g_free(key);
1370             g_free(self->cached_key);
1371             self->cached_key = NULL;
1372             g_free(self->cached_buf);
1373             self->cached_buf = NULL;
1374
1375             pself->block++;
1376             return *size_req;
1377         } else {
1378             *size_req = self->cached_size;
1379             g_free(key);
1380             return 0;
1381         }
1382     }
1383
1384     /* clear the cache, as it's useless to us */
1385     if (self->cached_key) {
1386         g_free(self->cached_key);
1387         self->cached_key = NULL;
1388
1389         g_free(self->cached_buf);
1390         self->cached_buf = NULL;
1391     }
1392
1393     /* set up dat for the write_func callback */
1394     if (!data || *size_req <= 0) {
1395         dat.data = NULL;
1396         dat.size_req = 0;
1397     } else {
1398         dat.data = data;
1399         dat.size_req = *size_req;
1400     }
1401
1402     result = s3_read(self->s3, self->bucket, key, s3_read_block_write_func,
1403         s3_buffer_reset_func, &dat, NULL, NULL);
1404     if (!result) {
1405         guint response_code;
1406         s3_error_code_t s3_error_code;
1407         s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1408
1409         g_free(key);
1410         key = NULL;
1411
1412         /* if it's an expected error (not found), just return -1 */
1413         if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1414             pself->is_eof = TRUE;
1415             pself->in_file = FALSE;
1416             device_set_error(pself,
1417                 stralloc(_("EOF")),
1418                 DEVICE_STATUS_SUCCESS);
1419             return -1;
1420         }
1421
1422         /* otherwise, log it and return FALSE */
1423         device_set_error(pself,
1424             vstrallocf(_("While reading data block from S3: %s"), s3_strerror(self->s3)),
1425             DEVICE_STATUS_VOLUME_ERROR);
1426         return -1;
1427     }
1428
1429     if (dat.data == NULL) {
1430         /* data was larger than the available space, so cache it and return
1431          * the actual size */
1432         self->cached_buf = dat.curl.buffer;
1433         self->cached_size = dat.curl.buffer_pos;
1434         self->cached_key = key;
1435         key = NULL;
1436
1437         *size_req = dat.curl.buffer_pos;
1438         return 0;
1439     }
1440
1441     /* ok, the read went directly to the user's buffer, so we need only
1442      * set and return the size */
1443     pself->block++;
1444     g_free(key);
1445     *size_req = dat.size_req;
1446     return dat.size_req;
1447 }