2 * Copyright (c) 2005 Zmanda, Inc. All Rights Reserved.
4 * This library is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU Lesser General Public License version 2.1 as
6 * published by the Free Software Foundation.
8 * This library is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
13 * You should have received a copy of the GNU Lesser General Public License
14 * along with this library; if not, write to the Free Software Foundation,
15 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
18 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
22 * data. It stores data in keys named with a user-specified prefix, inside a
23 * user-specified bucket. Data is stored in the form of numbered (large)
28 #include <sys/types.h>
38 #include "s3-device.h"
39 #include <curl/curl.h>
40 #ifdef HAVE_OPENSSL_HMAC_H
41 # include <openssl/hmac.h>
43 # ifdef HAVE_CRYPTO_HMAC_H
44 # include <crypto/hmac.h>
53 * Constants and static data
56 /* Maximum key length as specified in the S3 documentation
57 * (*excluding* null terminator) */
58 #define S3_MAX_KEY_LENGTH 1024
60 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
61 #define S3_DEVICE_MAX_BLOCK_SIZE (10*1024*1024)
63 /* This goes in lieu of file number for metadata. */
64 #define SPECIAL_INFIX "special-"
66 /* pointer to the class of our parent */
67 static DeviceClass *parent_class = NULL;
74 * utility functions */
76 /* Given file and block numbers, return an S3 key.
78 * @param self: the S3Device object
79 * @param file: the file number
80 * @param block: the block within that file
81 * @returns: a newly allocated string containing an S3 key.
84 file_and_block_to_key(S3Device *self,
88 /* Given the name of a special file (such as 'tapestart'), generate
89 * the S3 key to use for that file.
91 * @param self: the S3Device object
92 * @param special_name: name of the special file
93 * @param file: a file number to include; omitted if -1
94 * @returns: a newly alocated string containing an S3 key.
97 special_file_to_key(S3Device *self,
100 /* Write an amanda header file to S3.
102 * @param self: the S3Device object
103 * @param label: the volume label
104 * @param timestamp: the volume timestamp
107 write_amanda_header(S3Device *self,
111 /* "Fast forward" this device to the end by looking up the largest file number
112 * present and setting the current file number one greater.
114 * @param self: the S3Device object
117 seek_to_end(S3Device *self);
119 /* Find the number of the last file that contains any data (even just a header).
121 * @param self: the S3Device object
122 * @returns: the last file, or -1 in event of an error
125 find_last_file(S3Device *self);
127 /* Delete all blocks in the given file, including the filestart block
129 * @param self: the S3Device object
130 * @param file: the file to delete
133 delete_file(S3Device *self,
136 /* Set up self->s3 as best as possible. Unless SILENT is TRUE,
137 * any problems will generate warnings (with g_warning). Regardless,
138 * the return value is TRUE iff self->s3 is useable.
140 * @param self: the S3Device object
141 * @param silent: silence warnings
142 * @returns: TRUE if the handle is set up
145 setup_handle(S3Device * self,
146 gboolean ignore_problems);
152 s3_device_init(S3Device * o);
155 s3_device_class_init(S3DeviceClass * c);
158 s3_device_finalize(GObject * o);
161 s3_device_factory(char * device_type,
165 * virtual functions */
168 s3_device_open_device(Device *pself,
171 static ReadLabelStatusFlags s3_device_read_label(Device * self);
174 s3_device_start(Device * self,
175 DeviceAccessMode mode,
180 s3_device_start_file(Device * self,
181 const dumpfile_t * jobInfo);
184 s3_device_write_block(Device * self,
190 s3_device_finish_file(Device * self);
193 s3_device_seek_file(Device *pself,
197 s3_device_seek_block(Device *pself,
201 s3_device_read_block(Device * pself,
206 s3_device_recycle_file(Device *pself,
209 static gboolean s3_device_property_set(Device * p_self, DevicePropertyId id,
211 static gboolean s3_device_property_get(Device * p_self, DevicePropertyId id,
217 /* {{{ file_and_block_to_key */
219 file_and_block_to_key(S3Device *self,
223 char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
224 self->prefix, file, (long long unsigned int)block);
225 g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
230 /* {{{ special_file_to_key */
232 special_file_to_key(S3Device *self,
237 return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
239 return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
243 /* {{{ write_amanda_header */
245 write_amanda_header(S3Device *self,
249 char * amanda_header = NULL;
252 gboolean header_fits, result;
253 dumpfile_t * dumpinfo = NULL;
255 /* build the header */
256 dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
257 amanda_header = device_build_amanda_header(DEVICE(self), dumpinfo,
258 &header_size, &header_fits);
261 _("Amanda tapestart header won't fit in a single block!\n"));
262 g_free(amanda_header);
266 /* write out the header and flush the uploads. */
267 key = special_file_to_key(self, "tapestart", -1);
268 result = s3_upload(self->s3, self->bucket, key, amanda_header, header_size);
269 g_free(amanda_header);
273 fprintf(stderr, _("While writing amanda header: %s\n"),
274 s3_strerror(self->s3));
280 /* {{{ seek_to_end */
282 seek_to_end(S3Device *self) {
285 Device *pself = DEVICE(self);
287 last_file = find_last_file(self);
291 pself->file = last_file;
297 /* Convert an object name into a file number, assuming the given prefix
298 * length. Returns -1 if the object name is invalid, or 0 if the object name
299 * is a "special" key. */
300 static int key_to_file(guint prefix_len, const char * key) {
304 /* skip the prefix */
305 g_return_val_if_fail(strlen(key) > prefix_len, -1);
309 if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
313 /* check that key starts with 'f' */
314 g_return_val_if_fail(key[0] == 'f', -1);
317 /* check that key is of the form "%08x-" */
318 for (i = 0; i < 8; i++) {
319 if (!(key[i] >= '0' && key[i] <= '9') &&
320 !(key[i] >= 'a' && key[i] <= 'f') &&
321 !(key[i] >= 'A' && key[i] <= 'F')) break;
323 if (key[i] != '-') return -1;
324 if (i < 8) return -1;
326 /* convert the file number */
328 file = strtoul(key, NULL, 16);
330 g_warning(_("unparseable file number '%s'"), key);
337 /* {{{ find_last_file */
338 /* Find the number of the last file that contains any data (even just a header).
339 * Returns -1 in event of an error
342 find_last_file(S3Device *self) {
345 unsigned int prefix_len = strlen(self->prefix);
348 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
349 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
351 fprintf(stderr, _("While listing S3 keys: %s\n"),
352 s3_strerror(self->s3));
356 for (; keys; keys = g_slist_remove(keys, keys->data)) {
357 int file = key_to_file(prefix_len, keys->data);
359 /* and if it's the last, keep it */
360 if (file > last_file)
368 /* {{{ find_next_file */
369 /* Find the number of the file following the requested one, if any.
370 * Returns 0 if there is no such file or -1 in event of an error
373 find_next_file(S3Device *self, int last_file) {
376 unsigned int prefix_len = strlen(self->prefix);
379 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
380 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
382 fprintf(stderr, _("While listing S3 keys: %s\n"),
383 s3_strerror(self->s3));
387 for (; keys; keys = g_slist_remove(keys, keys->data)) {
390 file = key_to_file(prefix_len, (char*)keys->data);
393 /* Set this in case we don't find a next file; this is not a
394 * hard error, so if we can find a next file we'll return that
399 if (file < next_file && file > last_file) {
408 /* {{{ delete_file */
410 delete_file(S3Device *self,
415 char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
417 result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
419 fprintf(stderr, _("While listing S3 keys: %s\n"),
420 s3_strerror(self->s3));
424 /* this will likely be a *lot* of keys */
425 for (; keys; keys = g_slist_remove(keys, keys->data)) {
426 if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
427 if (!s3_delete(self->s3, self->bucket, keys->data)) {
428 fprintf(stderr, _("While deleting key '%s': %s\n"),
429 (char*)keys->data, s3_strerror(self->s3));
443 /* {{{ s3_device_register */
445 s3_device_register(void)
447 static const char * device_prefix_list[] = { "s3", NULL };
449 register_device(s3_device_factory, device_prefix_list);
453 /* {{{ s3_device_get_type */
455 s3_device_get_type(void)
457 static GType type = 0;
459 if G_UNLIKELY(type == 0) {
460 static const GTypeInfo info = {
461 sizeof (S3DeviceClass),
462 (GBaseInitFunc) NULL,
463 (GBaseFinalizeFunc) NULL,
464 (GClassInitFunc) s3_device_class_init,
465 (GClassFinalizeFunc) NULL,
466 NULL /* class_data */,
469 (GInstanceInitFunc) s3_device_init,
473 type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
481 /* {{{ s3_device_init */
483 s3_device_init(S3Device * self)
489 self->initializing = TRUE;
491 /* Register property values */
493 bzero(&response, sizeof(response));
495 prop.base = &device_property_concurrency;
496 prop.access = PROPERTY_ACCESS_GET_MASK;
497 g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
498 g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
499 device_add_property(o, &prop, &response);
500 g_value_unset(&response);
502 prop.base = &device_property_streaming;
503 g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
504 g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
505 device_add_property(o, &prop, &response);
506 g_value_unset(&response);
508 prop.base = &device_property_block_size;
509 g_value_init(&response, G_TYPE_INT);
510 g_value_set_int(&response, -1); /* indicates a variable block size; see below */
511 device_add_property(o, &prop, &response);
512 g_value_unset(&response);
514 prop.base = &device_property_min_block_size;
515 g_value_init(&response, G_TYPE_UINT);
516 g_value_set_uint(&response, S3_DEVICE_MIN_BLOCK_SIZE);
517 device_add_property(o, &prop, &response);
519 prop.base = &device_property_max_block_size;
520 g_value_set_uint(&response, S3_DEVICE_MAX_BLOCK_SIZE);
521 device_add_property(o, &prop, &response);
522 g_value_unset(&response);
524 prop.base = &device_property_appendable;
525 g_value_init(&response, G_TYPE_BOOLEAN);
526 g_value_set_boolean(&response, TRUE);
527 device_add_property(o, &prop, &response);
529 prop.base = &device_property_partial_deletion;
530 g_value_set_boolean(&response, TRUE);
531 device_add_property(o, &prop, &response);
532 g_value_unset(&response);
534 prop.base = &device_property_canonical_name;
535 g_value_init(&response, G_TYPE_STRING);
536 g_value_set_static_string(&response, "s3:");
537 device_add_property(o, &prop, &response);
538 g_value_unset(&response);
540 prop.base = &device_property_medium_access_type;
541 g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
542 g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
543 device_add_property(o, &prop, &response);
544 g_value_unset(&response);
546 prop.access = PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START;
547 prop.base = &device_property_s3_secret_key;
548 device_add_property(o, &prop, NULL);
549 prop.base = &device_property_s3_access_key;
550 device_add_property(o, &prop, NULL);
552 prop.base = &device_property_s3_user_token;
553 device_add_property(o, &prop, NULL);
558 /* {{{ s3_device_class_init */
560 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
562 GObjectClass *g_object_class = (GObjectClass*) c;
563 DeviceClass *device_class = (DeviceClass *)c;
565 parent_class = g_type_class_ref (TYPE_DEVICE);
567 device_class->open_device = s3_device_open_device;
568 device_class->read_label = s3_device_read_label;
569 device_class->start = s3_device_start;
571 device_class->start_file = s3_device_start_file;
572 device_class->write_block = s3_device_write_block;
573 device_class->finish_file = s3_device_finish_file;
575 device_class->seek_file = s3_device_seek_file;
576 device_class->seek_block = s3_device_seek_block;
577 device_class->read_block = s3_device_read_block;
578 device_class->recycle_file = s3_device_recycle_file;
580 device_class->property_set = s3_device_property_set;
581 device_class->property_get = s3_device_property_get;
583 g_object_class->finalize = s3_device_finalize;
587 /* {{{ s3_device_factory */
589 s3_device_factory(char * device_type,
594 g_assert(0 == strcmp(device_type, "s3"));
595 rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
596 s3_rval = (S3Device*)rval;
598 if (!device_open_device(rval, device_name)) {
599 g_object_unref(rval);
602 s3_rval->initializing = FALSE;
610 * Virtual function overrides
613 /* {{{ s3_device_open_device */
615 s3_device_open_device(Device *pself,
618 S3Device *self = S3_DEVICE(pself);
621 g_return_val_if_fail(self != NULL, FALSE);
623 /* Device name may be bucket/prefix, to support multiple volumes in a
625 name_colon = index(device_name, '/');
626 if (name_colon == NULL) {
627 self->bucket = g_strdup(device_name);
628 self->prefix = g_strdup("");
630 self->bucket = g_strndup(device_name, name_colon - device_name);
631 self->prefix = g_strdup(name_colon + 1);
634 if (self->bucket == NULL || self->bucket[0] == '\0') {
635 fprintf(stderr, _("Empty bucket name in device %s.\n"), device_name);
636 amfree(self->bucket);
637 amfree(self->prefix);
641 g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
644 self->verbose = FALSE;
646 if (parent_class->open_device) {
647 parent_class->open_device(pself, device_name);
654 /* {{{ s3_device_finalize */
655 static void s3_device_finalize(GObject * obj_self) {
656 S3Device *self = S3_DEVICE (obj_self);
658 if(G_OBJECT_CLASS(parent_class)->finalize)
659 (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
661 if(self->s3) s3_free(self->s3);
662 if(self->bucket) g_free(self->bucket);
663 if(self->prefix) g_free(self->prefix);
667 static gboolean setup_handle(S3Device * self, G_GNUC_UNUSED gboolean silent) {
668 if (self->s3 == NULL) {
669 if (self->access_key == NULL) {
670 if (!silent) fprintf(stderr, _("No S3 access key specified\n"));
673 if (self->secret_key == NULL) {
674 if (!silent) fprintf(stderr, _("No S3 secret key specified\n"));
678 if (self->user_token == NULL) {
679 if (!silent) fprintf(stderr, _("No S3 user token specified\n"));
683 self->s3 = s3_open(self->access_key, self->secret_key
688 if (self->s3 == NULL) {
689 fprintf(stderr, "Internal error creating S3 handle.\n");
694 s3_verbose(self->s3, self->verbose);
699 /* {{{ s3_device_read_label */
700 static ReadLabelStatusFlags
701 s3_device_read_label(Device *pself) {
702 S3Device *self = S3_DEVICE(pself);
706 dumpfile_t amanda_header;
708 if (!setup_handle(self, self->initializing))
709 return READ_LABEL_STATUS_DEVICE_ERROR;
711 key = special_file_to_key(self, "tapestart", -1);
712 if (!s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE)) {
714 s3_error_code_t s3_error_code;
715 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
717 /* if it's an expected error (not found), just return FALSE */
718 if (response_code == 404 &&
719 (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
720 g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
721 return READ_LABEL_STATUS_VOLUME_UNLABELED;
724 /* otherwise, log it and return */
725 fprintf(stderr, _("While trying to read tapestart header: %s\n"),
726 s3_strerror(self->s3));
727 return READ_LABEL_STATUS_DEVICE_ERROR;
730 g_assert(buf != NULL);
731 fh_init(&amanda_header);
732 parse_file_header(buf, &amanda_header, buf_size);
736 if (amanda_header.type != F_TAPESTART) {
737 fprintf(stderr, _("Invalid amanda header\n"));
738 return READ_LABEL_STATUS_VOLUME_ERROR;
741 amfree(pself->volume_label);
742 pself->volume_label = g_strdup(amanda_header.name);
743 amfree(pself->volume_time);
744 pself->volume_time = g_strdup(amanda_header.datestamp);
746 return READ_LABEL_STATUS_SUCCESS;
750 /* {{{ s3_device_start */
752 s3_device_start (Device * pself, DeviceAccessMode mode,
753 char * label, char * timestamp) {
757 self = S3_DEVICE(pself);
758 g_return_val_if_fail (self != NULL, FALSE);
760 if (!setup_handle(self, FALSE))
763 /* try creating the bucket, in case it doesn't exist */
764 if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
766 s3_error_code_t s3_error_code;
767 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
769 /* if it isn't an expected error (bucket already exists),
771 if (response_code != 409 ||
772 s3_error_code != S3_ERROR_BucketAlreadyExists) {
773 fprintf(stderr, _("While creating new S3 bucket: %s\n"),
774 s3_strerror(self->s3));
779 /* call up to the parent (Device) to set access_mode, volume_label,
780 * and volume_time, either from the arguments (ACCESS_WRITE) or by
781 * reading from the 0th file (otherwise)
783 if (parent_class->start)
784 if (!parent_class->start((Device*)self, mode, label, timestamp))
787 /* take care of any dirty work for this mode */
793 /* delete all files */
794 last_file = find_last_file(self);
795 if (last_file < 0) return FALSE;
796 for (file = 0; file <= last_file; file++) {
797 if (!delete_file(self, file)) return FALSE;
800 /* write a new amanda header */
801 if (!write_amanda_header(self, label, timestamp)) {
807 return seek_to_end(self);
810 g_assert_not_reached();
813 g_assert(pself->access_mode == mode);
819 static gboolean s3_device_property_get(Device * p_self, DevicePropertyId id,
822 const DevicePropertyBase * base;
824 self = S3_DEVICE(p_self);
825 g_return_val_if_fail(self != NULL, FALSE);
827 base = device_property_get_by_id(id);
828 g_return_val_if_fail(self != NULL, FALSE);
830 g_value_unset_init(val, base->type);
832 if (id == PROPERTY_S3_SECRET_KEY) {
833 if (self->secret_key != NULL) {
834 g_value_set_string(val, self->secret_key);
839 } else if (id == PROPERTY_S3_ACCESS_KEY) {
840 if (self->access_key != NULL) {
841 g_value_set_string(val, self->access_key);
848 else if (id == PROPERTY_S3_USER_TOKEN) {
849 if (self->user_token != NULL) {
850 g_value_set_string(val, self->user_token);
856 #endif /* WANT_DEVPAY */
857 else if (id == PROPERTY_VERBOSE) {
858 g_value_set_boolean(val, self->verbose);
862 if (parent_class->property_get) {
863 return (parent_class->property_get)(p_self, id, val);
869 g_assert_not_reached();
872 static gboolean s3_device_property_set(Device * p_self, DevicePropertyId id,
875 const DevicePropertyBase * base;
877 self = S3_DEVICE(p_self);
878 g_return_val_if_fail(self != NULL, FALSE);
880 base = device_property_get_by_id(id);
881 g_return_val_if_fail(self != NULL, FALSE);
883 g_return_val_if_fail(G_VALUE_HOLDS(val, base->type), FALSE);
885 if (id == PROPERTY_S3_SECRET_KEY) {
886 if (p_self->access_mode != ACCESS_NULL)
888 amfree(self->secret_key);
889 self->secret_key = g_value_dup_string(val);
890 device_clear_volume_details(p_self);
892 } else if (id == PROPERTY_S3_ACCESS_KEY) {
893 if (p_self->access_mode != ACCESS_NULL)
895 amfree(self->access_key);
896 self->access_key = g_value_dup_string(val);
897 device_clear_volume_details(p_self);
901 else if (id == PROPERTY_S3_USER_TOKEN) {
902 if (p_self->access_mode != ACCESS_NULL)
904 amfree(self->user_token);
905 self->user_token = g_value_dup_string(val);
906 device_clear_volume_details(p_self);
909 #endif /* WANT_DEVPAY */
910 else if (id == PROPERTY_VERBOSE) {
911 self->verbose = g_value_get_boolean(val);
912 /* Our S3 handle may not yet have been instantiated; if so, it will
913 * get the proper verbose setting when it is created */
915 s3_verbose(self->s3, self->verbose);
918 if (parent_class->property_set) {
919 return (parent_class->property_set)(p_self, id, val);
925 g_assert_not_reached();
928 /* functions for writing */
930 /* {{{ s3_device_start_file */
933 s3_device_start_file (Device *pself, const dumpfile_t *jobInfo) {
934 S3Device *self = S3_DEVICE(pself);
937 gboolean header_fits, result;
940 g_return_val_if_fail (self != NULL, FALSE);
942 /* Build the amanda header. */
943 amanda_header = device_build_amanda_header(pself, jobInfo,
944 &header_size, &header_fits);
945 g_return_val_if_fail(amanda_header != NULL, FALSE);
946 g_return_val_if_fail(header_fits, FALSE);
948 /* set the file and block numbers correctly */
949 pself->file = (pself->file > 0)? pself->file+1 : 1;
951 pself->in_file = TRUE;
953 /* write it out as a special block (not the 0th) */
954 key = special_file_to_key(self, "filestart", pself->file);
955 result = s3_upload(self->s3, self->bucket, key, amanda_header, header_size);
956 g_free(amanda_header);
959 fprintf(stderr, _("While writing filestart header: %s\n"),
960 s3_strerror(self->s3));
968 /* {{{ s3_device_write_block */
970 s3_device_write_block (Device * pself, guint size, gpointer data,
971 gboolean last_block) {
974 S3Device * self = S3_DEVICE(pself);;
976 g_assert (self != NULL);
977 g_assert (data != NULL);
979 filename = file_and_block_to_key(self, pself->file, pself->block);
981 result = s3_upload(self->s3, self->bucket, filename, data, size);
984 fprintf(stderr, _("While writing data block to S3: %s\n"),
985 s3_strerror(self->s3));
991 /* if this is the last block, finish the file */
993 return s3_device_finish_file(pself);
1000 /* {{{ s3_device_finish_file */
1002 s3_device_finish_file (Device * pself) {
1003 /* we're not in a file anymore */
1004 pself->in_file = FALSE;
1010 /* {{{ s3_device_recycle_file */
1012 s3_device_recycle_file(Device *pself, guint file) {
1013 S3Device *self = S3_DEVICE(pself);
1015 return delete_file(self, file);
1019 /* functions for reading */
1021 /* {{{ s3_device_seek_file */
1023 s3_device_seek_file(Device *pself, guint file) {
1024 S3Device *self = S3_DEVICE(pself);
1029 dumpfile_t *amanda_header;
1033 pself->in_file = TRUE;
1036 key = special_file_to_key(self, "filestart", pself->file);
1037 result = s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE);
1041 guint response_code;
1042 s3_error_code_t s3_error_code;
1043 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1045 /* if it's an expected error (not found), check what to do. */
1046 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1049 pself->in_file = FALSE;
1050 next_file = find_next_file(self, pself->file);
1051 if (next_file > 0) {
1052 /* Note short-circut of dispatcher. */
1053 return s3_device_seek_file(pself, next_file);
1054 } else if (next_file == 0) {
1055 /* No next file. Check if we are one past the end. */
1056 key = special_file_to_key(self, "filestart", pself->file - 1);
1057 result = s3_read(self->s3, self->bucket, key, &buf, &buf_size,
1058 S3_DEVICE_MAX_BLOCK_SIZE);
1061 return make_tapeend_header();
1067 /* An error occured finding out if we are the last file. */
1072 /* and make a dumpfile_t out of it */
1073 g_assert(buf != NULL);
1074 amanda_header = g_new(dumpfile_t, 1);
1075 fh_init(amanda_header);
1076 parse_file_header(buf, amanda_header, buf_size);
1079 switch (amanda_header->type) {
1081 case F_CONT_DUMPFILE:
1082 case F_SPLIT_DUMPFILE:
1083 return amanda_header;
1087 _("Invalid amanda header while reading file header\n"));
1088 g_free(amanda_header);
1094 /* {{{ s3_device_seek_block */
1096 s3_device_seek_block(Device *pself, guint64 block) {
1097 pself->block = block;
1102 /* {{{ s3_device_read_block */
1104 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1105 S3Device * self = S3_DEVICE(pself);
1111 g_assert (self != NULL);
1114 key = file_and_block_to_key(self, pself->file, pself->block);
1115 g_assert(key != NULL);
1116 if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1117 /* use the cached copy and clear the cache */
1118 buf = self->cached_buf;
1119 buf_size = self->cached_size;
1121 self->cached_buf = NULL;
1122 g_free(self->cached_key);
1123 self->cached_key = NULL;
1125 /* clear the cache and actually download the file */
1126 if (self->cached_buf) {
1127 g_free(self->cached_buf);
1128 self->cached_buf = NULL;
1130 if (self->cached_key) {
1131 g_free(self->cached_key);
1132 self->cached_key = NULL;
1135 result = s3_read(self->s3, self->bucket, key, &buf, &buf_size, S3_DEVICE_MAX_BLOCK_SIZE);
1137 guint response_code;
1138 s3_error_code_t s3_error_code;
1139 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1144 /* if it's an expected error (not found), just return -1 */
1145 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1146 pself->is_eof = TRUE;
1150 /* otherwise, log it and return FALSE */
1151 fprintf(stderr, _("While reading data block from S3: %s\n"),
1152 s3_strerror(self->s3));
1157 /* INVARIANT: cache is NULL */
1158 g_assert(self->cached_buf == NULL);
1159 g_assert(self->cached_key == NULL);
1161 /* now see how the caller wants to deal with that */
1162 if (data == NULL || *size_req < 0 || buf_size > (guint)*size_req) {
1163 /* A size query or short buffer -- load the cache and return the size*/
1164 self->cached_buf = buf;
1165 self->cached_key = key;
1166 self->cached_size = buf_size;
1168 *size_req = buf_size;
1171 /* ok, all checks are passed -- copy the data */
1172 *size_req = buf_size;
1173 g_memmove(data, buf, buf_size);
1177 /* move on to the next block */