2 * Copyright (c) 2005-2008 Zmanda Inc. All Rights Reserved.
4 * This library is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU Lesser General Public License version 2.1 as
6 * published by the Free Software Foundation.
8 * This library is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
13 * You should have received a copy of the GNU Lesser General Public License
14 * along with this library; if not, write to the Free Software Foundation,
15 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 * Contact information: Zmanda Inc., 465 S Mathlida Ave, Suite 300
18 * Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
22 * data. It stores data in keys named with a user-specified prefix, inside a
23 * user-specified bucket. Data is stored in the form of numbered (large)
28 #include <sys/types.h>
39 #include <curl/curl.h>
40 #ifdef HAVE_OPENSSL_HMAC_H
41 # include <openssl/hmac.h>
43 # ifdef HAVE_CRYPTO_HMAC_H
44 # include <crypto/hmac.h>
53 * Type checking and casting macros
55 #define TYPE_S3_DEVICE (s3_device_get_type())
56 #define S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
57 #define S3_DEVICE_CONST(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
58 #define S3_DEVICE_CLASS(klass) G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
59 #define IS_S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
61 #define S3_DEVICE_GET_CLASS(obj) G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
62 static GType s3_device_get_type (void);
65 * Main object structure
67 typedef struct _S3MetadataFile S3MetadataFile;
69 typedef struct _S3Device S3Device;
73 /* The "easy" curl handle we use to access Amazon S3 */
76 /* S3 access information */
80 /* The S3 access information. */
86 char *bucket_location;
88 /* a cache for unsuccessful reads (where we get the file but the caller
89 * doesn't have space for it or doesn't want it), where we expect the
90 * next call will request the same file.
96 /* Produce verbose output? */
106 typedef struct _S3DeviceClass S3DeviceClass;
107 struct _S3DeviceClass {
108 DeviceClass __parent__;
113 * Constants and static data
116 #define S3_DEVICE_NAME "s3"
117 #define DEVPAY_DEVICE_NAME "s3zmanda"
119 /* Maximum key length as specified in the S3 documentation
120 * (*excluding* null terminator) */
121 #define S3_MAX_KEY_LENGTH 1024
123 /* Note: for compatability, min can only be decreased and max increased */
124 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
125 #define S3_DEVICE_MAX_BLOCK_SIZE (100*1024*1024)
126 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
128 /* This goes in lieu of file number for metadata. */
129 #define SPECIAL_INFIX "special-"
131 /* pointer to the class of our parent */
132 static DeviceClass *parent_class = NULL;
135 * device-specific properties
138 /* Authentication information for Amazon S3. Both of these are strings. */
139 static DevicePropertyBase device_property_s3_access_key;
140 static DevicePropertyBase device_property_s3_secret_key;
141 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
142 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
144 /* Same, but for S3 with DevPay. */
145 static DevicePropertyBase device_property_s3_user_token;
146 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
148 /* Location constraint for new buckets created on Amazon S3. */
149 static DevicePropertyBase device_property_s3_bucket_location;
150 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
152 /* Whether to use SSL with Amazon S3. */
153 static DevicePropertyBase device_property_s3_ssl;
154 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
161 void s3_device_register(void);
164 * utility functions */
166 /* Given file and block numbers, return an S3 key.
168 * @param self: the S3Device object
169 * @param file: the file number
170 * @param block: the block within that file
171 * @returns: a newly allocated string containing an S3 key.
174 file_and_block_to_key(S3Device *self,
178 /* Given the name of a special file (such as 'tapestart'), generate
179 * the S3 key to use for that file.
181 * @param self: the S3Device object
182 * @param special_name: name of the special file
183 * @param file: a file number to include; omitted if -1
184 * @returns: a newly alocated string containing an S3 key.
187 special_file_to_key(S3Device *self,
190 /* Write an amanda header file to S3.
192 * @param self: the S3Device object
193 * @param label: the volume label
194 * @param timestamp: the volume timestamp
197 write_amanda_header(S3Device *self,
201 /* "Fast forward" this device to the end by looking up the largest file number
202 * present and setting the current file number one greater.
204 * @param self: the S3Device object
207 seek_to_end(S3Device *self);
209 /* Find the number of the last file that contains any data (even just a header).
211 * @param self: the S3Device object
212 * @returns: the last file, or -1 in event of an error
215 find_last_file(S3Device *self);
217 /* Delete all blocks in the given file, including the filestart block
219 * @param self: the S3Device object
220 * @param file: the file to delete
223 delete_file(S3Device *self,
226 /* Set up self->s3 as best as possible.
228 * The return value is TRUE iff self->s3 is useable.
230 * @param self: the S3Device object
231 * @returns: TRUE if the handle is set up
234 setup_handle(S3Device * self);
240 s3_device_init(S3Device * o);
243 s3_device_class_init(S3DeviceClass * c);
246 s3_device_finalize(GObject * o);
249 s3_device_factory(char * device_name, char * device_type, char * device_node);
252 * Property{Get,Set}Fns */
254 static gboolean s3_device_set_access_key_fn(Device *self,
255 DevicePropertyBase *base, GValue *val,
256 PropertySurety surety, PropertySource source);
258 static gboolean s3_device_set_secret_key_fn(Device *self,
259 DevicePropertyBase *base, GValue *val,
260 PropertySurety surety, PropertySource source);
262 static gboolean s3_device_set_user_token_fn(Device *self,
263 DevicePropertyBase *base, GValue *val,
264 PropertySurety surety, PropertySource source);
266 static gboolean s3_device_set_bucket_location_fn(Device *self,
267 DevicePropertyBase *base, GValue *val,
268 PropertySurety surety, PropertySource source);
270 static gboolean s3_device_set_verbose_fn(Device *self,
271 DevicePropertyBase *base, GValue *val,
272 PropertySurety surety, PropertySource source);
274 static gboolean s3_device_set_ssl_fn(Device *self,
275 DevicePropertyBase *base, GValue *val,
276 PropertySurety surety, PropertySource source);
279 * virtual functions */
282 s3_device_open_device(Device *pself, char *device_name,
283 char * device_type, char * device_node);
285 static DeviceStatusFlags s3_device_read_label(Device * self);
288 s3_device_start(Device * self,
289 DeviceAccessMode mode,
294 s3_device_finish(Device * self);
297 s3_device_start_file(Device * self,
298 dumpfile_t * jobInfo);
301 s3_device_write_block(Device * self,
306 s3_device_finish_file(Device * self);
309 s3_device_seek_file(Device *pself,
313 s3_device_seek_block(Device *pself,
317 s3_device_read_block(Device * pself,
322 s3_device_recycle_file(Device *pself,
330 file_and_block_to_key(S3Device *self,
334 char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
335 self->prefix, file, (long long unsigned int)block);
336 g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
341 special_file_to_key(S3Device *self,
346 return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
348 return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
352 write_amanda_header(S3Device *self,
356 CurlBuffer amanda_header = {NULL, 0, 0, 0};
358 gboolean header_fits, result;
359 dumpfile_t * dumpinfo = NULL;
360 Device *d_self = DEVICE(self);
362 /* build the header */
363 dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
364 amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
365 /* casting guint* to int* */
366 (int*) &amanda_header.buffer_len, &header_fits);
368 device_set_error(d_self,
369 stralloc(_("Amanda tapestart header won't fit in a single block!")),
370 DEVICE_STATUS_DEVICE_ERROR);
371 g_free(amanda_header.buffer);
375 /* write out the header and flush the uploads. */
376 key = special_file_to_key(self, "tapestart", -1);
377 result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
378 &amanda_header, NULL, NULL);
379 g_free(amanda_header.buffer);
383 device_set_error(d_self,
384 vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3)),
385 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
391 seek_to_end(S3Device *self) {
394 Device *pself = DEVICE(self);
396 last_file = find_last_file(self);
400 pself->file = last_file;
405 /* Convert an object name into a file number, assuming the given prefix
406 * length. Returns -1 if the object name is invalid, or 0 if the object name
407 * is a "special" key. */
408 static int key_to_file(guint prefix_len, const char * key) {
412 /* skip the prefix */
413 if (strlen(key) <= prefix_len)
418 if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
422 /* check that key starts with 'f' */
427 /* check that key is of the form "%08x-" */
428 for (i = 0; i < 8; i++) {
429 if (!(key[i] >= '0' && key[i] <= '9') &&
430 !(key[i] >= 'a' && key[i] <= 'f') &&
431 !(key[i] >= 'A' && key[i] <= 'F')) break;
433 if (key[i] != '-') return -1;
434 if (i < 8) return -1;
436 /* convert the file number */
438 file = strtoul(key, NULL, 16);
440 g_warning(_("unparseable file number '%s'"), key);
447 /* Find the number of the last file that contains any data (even just a header).
448 * Returns -1 in event of an error
451 find_last_file(S3Device *self) {
454 unsigned int prefix_len = strlen(self->prefix);
456 Device *d_self = DEVICE(self);
458 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
459 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
461 device_set_error(d_self,
462 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
463 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
467 for (; keys; keys = g_slist_remove(keys, keys->data)) {
468 int file = key_to_file(prefix_len, keys->data);
470 /* and if it's the last, keep it */
471 if (file > last_file)
478 /* Find the number of the file following the requested one, if any.
479 * Returns 0 if there is no such file or -1 in event of an error
482 find_next_file(S3Device *self, int last_file) {
485 unsigned int prefix_len = strlen(self->prefix);
487 Device *d_self = DEVICE(self);
489 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
490 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
492 device_set_error(d_self,
493 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
494 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
498 for (; keys; keys = g_slist_remove(keys, keys->data)) {
501 file = key_to_file(prefix_len, (char*)keys->data);
504 /* Set this in case we don't find a next file; this is not a
505 * hard error, so if we can find a next file we'll return that
510 if (file < next_file && file > last_file) {
519 delete_file(S3Device *self,
524 char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
525 Device *d_self = DEVICE(self);
527 result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
529 device_set_error(d_self,
530 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
531 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
535 /* this will likely be a *lot* of keys */
536 for (; keys; keys = g_slist_remove(keys, keys->data)) {
537 if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
538 if (!s3_delete(self->s3, self->bucket, keys->data)) {
539 device_set_error(d_self,
540 vstrallocf(_("While deleting key '%s': %s"),
541 (char*)keys->data, s3_strerror(self->s3)),
542 DEVICE_STATUS_DEVICE_ERROR);
556 s3_device_register(void)
558 static const char * device_prefix_list[] = { S3_DEVICE_NAME, DEVPAY_DEVICE_NAME, NULL };
561 /* set up our properties */
562 device_property_fill_and_register(&device_property_s3_secret_key,
563 G_TYPE_STRING, "s3_secret_key",
564 "Secret access key to authenticate with Amazon S3");
565 device_property_fill_and_register(&device_property_s3_access_key,
566 G_TYPE_STRING, "s3_access_key",
567 "Access key ID to authenticate with Amazon S3");
568 device_property_fill_and_register(&device_property_s3_user_token,
569 G_TYPE_STRING, "s3_user_token",
570 "User token for authentication Amazon devpay requests");
571 device_property_fill_and_register(&device_property_s3_bucket_location,
572 G_TYPE_STRING, "s3_bucket_location",
573 "Location constraint for buckets on Amazon S3");
574 device_property_fill_and_register(&device_property_s3_ssl,
575 G_TYPE_BOOLEAN, "s3_ssl",
576 "Whether to use SSL with Amazon S3");
579 /* register the device itself */
580 register_device(s3_device_factory, device_prefix_list);
584 s3_device_get_type(void)
586 static GType type = 0;
588 if G_UNLIKELY(type == 0) {
589 static const GTypeInfo info = {
590 sizeof (S3DeviceClass),
591 (GBaseInitFunc) NULL,
592 (GBaseFinalizeFunc) NULL,
593 (GClassInitFunc) s3_device_class_init,
594 (GClassFinalizeFunc) NULL,
595 NULL /* class_data */,
598 (GInstanceInitFunc) s3_device_init,
602 type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
610 s3_device_init(S3Device * self)
612 Device * dself = DEVICE(self);
615 /* Register property values
616 * Note: Some aren't added until s3_device_open_device()
618 bzero(&response, sizeof(response));
620 g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
621 g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
622 device_set_simple_property(dself, PROPERTY_CONCURRENCY,
623 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
624 g_value_unset(&response);
626 g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
627 g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
628 device_set_simple_property(dself, PROPERTY_STREAMING,
629 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
630 g_value_unset(&response);
632 g_value_init(&response, G_TYPE_BOOLEAN);
633 g_value_set_boolean(&response, TRUE);
634 device_set_simple_property(dself, PROPERTY_APPENDABLE,
635 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
636 g_value_unset(&response);
638 g_value_init(&response, G_TYPE_BOOLEAN);
639 g_value_set_boolean(&response, TRUE);
640 device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
641 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
642 g_value_unset(&response);
644 g_value_init(&response, G_TYPE_BOOLEAN);
645 g_value_set_boolean(&response, FALSE);
646 device_set_simple_property(dself, PROPERTY_COMPRESSION,
647 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
648 g_value_unset(&response);
650 g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
651 g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
652 device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
653 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
654 g_value_unset(&response);
659 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
661 GObjectClass *g_object_class = (GObjectClass*) c;
662 DeviceClass *device_class = (DeviceClass *)c;
664 parent_class = g_type_class_ref (TYPE_DEVICE);
666 device_class->open_device = s3_device_open_device;
667 device_class->read_label = s3_device_read_label;
668 device_class->start = s3_device_start;
669 device_class->finish = s3_device_finish;
671 device_class->start_file = s3_device_start_file;
672 device_class->write_block = s3_device_write_block;
673 device_class->finish_file = s3_device_finish_file;
675 device_class->seek_file = s3_device_seek_file;
676 device_class->seek_block = s3_device_seek_block;
677 device_class->read_block = s3_device_read_block;
678 device_class->recycle_file = s3_device_recycle_file;
680 g_object_class->finalize = s3_device_finalize;
682 device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
683 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
684 device_simple_property_get_fn,
685 s3_device_set_access_key_fn);
687 device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
688 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
689 device_simple_property_get_fn,
690 s3_device_set_secret_key_fn);
692 device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
693 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
694 device_simple_property_get_fn,
695 s3_device_set_user_token_fn);
697 device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
698 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
699 device_simple_property_get_fn,
700 s3_device_set_bucket_location_fn);
702 device_class_register_property(device_class, PROPERTY_VERBOSE,
703 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
704 device_simple_property_get_fn,
705 s3_device_set_verbose_fn);
707 device_class_register_property(device_class, PROPERTY_S3_SSL,
708 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
709 device_simple_property_get_fn,
710 s3_device_set_ssl_fn);
712 device_class_register_property(device_class, PROPERTY_COMPRESSION,
713 PROPERTY_ACCESS_GET_MASK,
714 device_simple_property_get_fn,
719 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
720 GValue *val, PropertySurety surety, PropertySource source)
722 S3Device *self = S3_DEVICE(p_self);
724 amfree(self->access_key);
725 self->access_key = g_value_dup_string(val);
726 device_clear_volume_details(p_self);
728 return device_simple_property_set_fn(p_self, base, val, surety, source);
732 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
733 GValue *val, PropertySurety surety, PropertySource source)
735 S3Device *self = S3_DEVICE(p_self);
737 amfree(self->secret_key);
738 self->secret_key = g_value_dup_string(val);
739 device_clear_volume_details(p_self);
741 return device_simple_property_set_fn(p_self, base, val, surety, source);
745 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
746 GValue *val, PropertySurety surety, PropertySource source)
748 S3Device *self = S3_DEVICE(p_self);
750 if (!self->is_devpay) {
751 device_set_error(p_self, stralloc(_(
752 "Can't set a user token unless DevPay is in use")),
753 DEVICE_STATUS_DEVICE_ERROR);
757 amfree(self->user_token);
758 self->user_token = g_value_dup_string(val);
759 device_clear_volume_details(p_self);
761 return device_simple_property_set_fn(p_self, base, val, surety, source);
765 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
766 GValue *val, PropertySurety surety, PropertySource source)
768 S3Device *self = S3_DEVICE(p_self);
770 if (self->use_ssl && !s3_curl_location_compat()) {
771 device_set_error(p_self, stralloc(_(
772 "Location constraint given for Amazon S3 bucket, "
773 "but libcurl is too old support wildcard certificates.")),
774 DEVICE_STATUS_DEVICE_ERROR);
778 if (!s3_bucket_location_compat(self->bucket)) {
779 device_set_error(p_self, g_strdup_printf(_(
780 "Location constraint given for Amazon S3 bucket, "
781 "but the bucket name (%s) is not usable as a subdomain."),
783 DEVICE_STATUS_DEVICE_ERROR);
787 amfree(self->bucket_location);
788 self->bucket_location = g_value_dup_string(val);
789 device_clear_volume_details(p_self);
791 return device_simple_property_set_fn(p_self, base, val, surety, source);
795 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
796 GValue *val, PropertySurety surety, PropertySource source)
798 S3Device *self = S3_DEVICE(p_self);
800 self->verbose = g_value_get_boolean(val);
801 /* Our S3 handle may not yet have been instantiated; if so, it will
802 * get the proper verbose setting when it is created */
804 s3_verbose(self->s3, self->verbose);
806 return device_simple_property_set_fn(p_self, base, val, surety, source);
810 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
811 GValue *val, PropertySurety surety, PropertySource source)
813 S3Device *self = S3_DEVICE(p_self);
816 new_val = g_value_get_boolean(val);
817 /* Our S3 handle may not yet have been instantiated; if so, it will
818 * get the proper use_ssl setting when it is created */
819 if (self->s3 && !s3_use_ssl(self->s3, new_val)) {
820 device_set_error(p_self, g_strdup_printf(_(
821 "Error setting S3 SSL/TLS use "
822 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
823 DEVICE_STATUS_DEVICE_ERROR);
826 self->use_ssl = new_val;
828 return device_simple_property_set_fn(p_self, base, val, surety, source);
832 s3_device_factory(char * device_name, char * device_type, char * device_node)
836 g_assert(0 == strcmp(device_type, S3_DEVICE_NAME) ||
837 0 == strcmp(device_type, DEVPAY_DEVICE_NAME));
838 rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
839 s3_rval = (S3Device*)rval;
841 device_open_device(rval, device_name, device_type, device_node);
846 * Virtual function overrides
850 s3_device_open_device(Device *pself, char *device_name,
851 char * device_type, char * device_node)
853 S3Device *self = S3_DEVICE(pself);
857 pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
858 pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
859 pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
861 /* Device name may be bucket/prefix, to support multiple volumes in a
863 name_colon = index(device_node, '/');
864 if (name_colon == NULL) {
865 self->bucket = g_strdup(device_node);
866 self->prefix = g_strdup("");
868 self->bucket = g_strndup(device_node, name_colon - device_node);
869 self->prefix = g_strdup(name_colon + 1);
872 self->is_devpay = !strcmp(device_type, DEVPAY_DEVICE_NAME);
874 if (self->bucket == NULL || self->bucket[0] == '\0') {
875 device_set_error(pself,
876 vstrallocf(_("Empty bucket name in device %s"), device_name),
877 DEVICE_STATUS_DEVICE_ERROR);
878 amfree(self->bucket);
879 amfree(self->prefix);
883 g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
886 self->verbose = FALSE;
888 /* use SSL if available */
889 self->use_ssl = s3_curl_supports_ssl();
890 bzero(&tmp_value, sizeof(GValue));
891 g_value_init(&tmp_value, G_TYPE_BOOLEAN);
892 g_value_set_boolean(&tmp_value, self->use_ssl);
893 device_set_simple_property(pself, device_property_s3_ssl.ID,
894 &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
896 if (parent_class->open_device) {
897 parent_class->open_device(pself, device_name, device_type, device_node);
901 static void s3_device_finalize(GObject * obj_self) {
902 S3Device *self = S3_DEVICE (obj_self);
904 if(G_OBJECT_CLASS(parent_class)->finalize)
905 (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
907 if(self->s3) s3_free(self->s3);
908 if(self->bucket) g_free(self->bucket);
909 if(self->prefix) g_free(self->prefix);
910 if(self->access_key) g_free(self->access_key);
911 if(self->secret_key) g_free(self->secret_key);
912 if(self->user_token) g_free(self->user_token);
913 if(self->bucket_location) g_free(self->bucket_location);
916 static gboolean setup_handle(S3Device * self) {
917 Device *d_self = DEVICE(self);
918 if (self->s3 == NULL) {
919 if (self->access_key == NULL)
921 if (self->secret_key == NULL)
923 if (self->is_devpay && self->user_token == NULL)
926 self->s3 = s3_open(self->access_key, self->secret_key, self->user_token,
927 self->bucket_location);
928 if (self->s3 == NULL) {
929 device_set_error(d_self,
930 stralloc(_("Internal error creating S3 handle")),
931 DEVICE_STATUS_DEVICE_ERROR);
936 s3_verbose(self->s3, self->verbose);
938 if (!s3_use_ssl(self->s3, self->use_ssl)) {
939 device_set_error(d_self, g_strdup_printf(_(
940 "Error setting S3 SSL/TLS use "
941 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
942 DEVICE_STATUS_DEVICE_ERROR);
949 static DeviceStatusFlags
950 s3_device_read_label(Device *pself) {
951 S3Device *self = S3_DEVICE(pself);
953 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
954 dumpfile_t *amanda_header;
956 /* note that this may be called from s3_device_start, when
957 * self->access_mode is not ACCESS_NULL */
959 amfree(pself->volume_label);
960 amfree(pself->volume_time);
961 amfree(pself->volume_header);
963 if (device_in_error(self)) return pself->status;
965 if (!setup_handle(self)) {
966 device_set_error(pself, stralloc(_("Error setting up S3 interface")), DEVICE_STATUS_DEVICE_ERROR);
967 return pself->status;
970 key = special_file_to_key(self, "tapestart", -1);
971 if (!s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
973 s3_error_code_t s3_error_code;
974 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
976 /* if it's an expected error (not found), just return FALSE */
977 if (response_code == 404 &&
978 (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
979 g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
980 device_set_error(pself,
981 stralloc(_("Amanda header not found -- unlabeled volume?")),
982 DEVICE_STATUS_DEVICE_ERROR
983 | DEVICE_STATUS_VOLUME_ERROR
984 | DEVICE_STATUS_VOLUME_UNLABELED);
985 return pself->status;
988 /* otherwise, log it and return */
989 device_set_error(pself,
990 vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3)),
991 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
992 return pself->status;
995 g_assert(buf.buffer != NULL);
996 amanda_header = g_new(dumpfile_t, 1);
997 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
998 pself->volume_header = amanda_header;
1002 if (amanda_header->type != F_TAPESTART) {
1003 device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
1004 return pself->status;
1007 pself->volume_label = g_strdup(amanda_header->name);
1008 pself->volume_time = g_strdup(amanda_header->datestamp);
1009 /* pself->volume_header is already set */
1011 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1013 return pself->status;
1017 s3_device_start (Device * pself, DeviceAccessMode mode,
1018 char * label, char * timestamp) {
1020 int file, last_file;
1022 self = S3_DEVICE(pself);
1024 if (device_in_error(self)) return FALSE;
1026 if (!setup_handle(self)) {
1027 device_set_error(pself,
1028 stralloc(_("Error setting up S3 interface")),
1029 DEVICE_STATUS_DEVICE_ERROR);
1033 pself->access_mode = mode;
1034 pself->in_file = FALSE;
1036 /* try creating the bucket, in case it doesn't exist */
1037 if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
1038 guint response_code;
1039 s3_error_code_t s3_error_code;
1040 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1042 /* if it isn't an expected error (bucket already exists),
1044 if (response_code != 409 ||
1045 s3_error_code != S3_ERROR_BucketAlreadyExists) {
1046 device_set_error(pself,
1047 vstrallocf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3)),
1048 DEVICE_STATUS_DEVICE_ERROR);
1053 /* take care of any dirty work for this mode */
1056 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1057 /* s3_device_read_label already set our error message */
1063 /* delete all files */
1064 last_file = find_last_file(self);
1065 if (last_file < 0) return FALSE;
1066 for (file = 0; file <= last_file; file++) {
1067 if (!delete_file(self, file))
1068 /* delete_file already set our error message */
1072 /* write a new amanda header */
1073 if (!write_amanda_header(self, label, timestamp)) {
1077 pself->volume_label = newstralloc(pself->volume_label, label);
1078 pself->volume_time = newstralloc(pself->volume_time, timestamp);
1080 /* unset the VOLUME_UNLABELED flag, if it was set */
1081 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1085 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1086 /* s3_device_read_label already set our error message */
1089 return seek_to_end(self);
1093 g_assert_not_reached();
1100 s3_device_finish (Device * pself) {
1101 if (device_in_error(pself)) return FALSE;
1103 /* we're not in a file anymore */
1104 pself->access_mode = ACCESS_NULL;
1109 /* functions for writing */
1113 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
1114 S3Device *self = S3_DEVICE(pself);
1115 CurlBuffer amanda_header = {NULL, 0, 0, 0};
1116 gboolean header_fits, result;
1119 if (device_in_error(self)) return FALSE;
1121 /* Set the blocksize to zero, since there's no header to skip (it's stored
1122 * in a distinct file, rather than block zero) */
1123 jobInfo->blocksize = 0;
1125 /* Build the amanda header. */
1126 amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
1127 (int *) &amanda_header.buffer_len, &header_fits);
1129 device_set_error(pself,
1130 stralloc(_("Amanda file header won't fit in a single block!")),
1131 DEVICE_STATUS_DEVICE_ERROR);
1135 /* set the file and block numbers correctly */
1136 pself->file = (pself->file > 0)? pself->file+1 : 1;
1138 pself->in_file = TRUE;
1140 /* write it out as a special block (not the 0th) */
1141 key = special_file_to_key(self, "filestart", pself->file);
1142 result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
1143 &amanda_header, NULL, NULL);
1144 g_free(amanda_header.buffer);
1147 device_set_error(pself,
1148 vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3)),
1149 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1157 s3_device_write_block (Device * pself, guint size, gpointer data) {
1160 S3Device * self = S3_DEVICE(pself);
1161 CurlBuffer to_write = {data, size, 0, 0};
1163 g_assert (self != NULL);
1164 g_assert (data != NULL);
1165 if (device_in_error(self)) return FALSE;
1167 filename = file_and_block_to_key(self, pself->file, pself->block);
1169 result = s3_upload(self->s3, self->bucket, filename, S3_BUFFER_READ_FUNCS,
1170 &to_write, NULL, NULL);
1173 device_set_error(pself,
1174 vstrallocf(_("While writing data block to S3: %s"), s3_strerror(self->s3)),
1175 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1185 s3_device_finish_file (Device * pself) {
1186 if (device_in_error(pself)) return FALSE;
1188 /* we're not in a file anymore */
1189 pself->in_file = FALSE;
1195 s3_device_recycle_file(Device *pself, guint file) {
1196 S3Device *self = S3_DEVICE(pself);
1197 if (device_in_error(self)) return FALSE;
1199 return delete_file(self, file);
1200 /* delete_file already set our error message if necessary */
1203 /* functions for reading */
1206 s3_device_seek_file(Device *pself, guint file) {
1207 S3Device *self = S3_DEVICE(pself);
1210 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1211 dumpfile_t *amanda_header;
1212 const char *errmsg = NULL;
1214 if (device_in_error(self)) return NULL;
1217 pself->is_eof = FALSE;
1218 pself->in_file = FALSE;
1222 key = special_file_to_key(self, "filestart", pself->file);
1223 result = s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
1228 guint response_code;
1229 s3_error_code_t s3_error_code;
1230 s3_error(self->s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
1232 /* if it's an expected error (not found), check what to do. */
1233 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1235 next_file = find_next_file(self, pself->file);
1236 if (next_file > 0) {
1237 /* Note short-circut of dispatcher. */
1238 return s3_device_seek_file(pself, next_file);
1239 } else if (next_file == 0) {
1240 /* No next file. Check if we are one past the end. */
1241 key = special_file_to_key(self, "filestart", pself->file - 1);
1242 result = s3_read(self->s3, self->bucket, key,
1243 S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
1246 /* pself->file, etc. are already correct */
1247 return make_tapeend_header();
1249 device_set_error(pself,
1250 stralloc(_("Attempt to read past tape-end file")),
1251 DEVICE_STATUS_SUCCESS);
1256 /* An unexpected error occured finding out if we are the last file. */
1257 device_set_error(pself,
1259 DEVICE_STATUS_DEVICE_ERROR);
1264 /* and make a dumpfile_t out of it */
1265 g_assert(buf.buffer != NULL);
1266 amanda_header = g_new(dumpfile_t, 1);
1267 fh_init(amanda_header);
1268 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1271 switch (amanda_header->type) {
1273 case F_CONT_DUMPFILE:
1274 case F_SPLIT_DUMPFILE:
1278 device_set_error(pself,
1279 stralloc(_("Invalid amanda header while reading file header")),
1280 DEVICE_STATUS_VOLUME_ERROR);
1281 g_free(amanda_header);
1285 pself->in_file = TRUE;
1286 return amanda_header;
1290 s3_device_seek_block(Device *pself, guint64 block) {
1291 if (device_in_error(pself)) return FALSE;
1293 pself->block = block;
1297 typedef struct s3_read_block_data {
1303 } s3_read_block_data;
1305 /* wrapper around s3_buffer_write_func to write as much data as possible to
1306 * the user's buffer, and switch to a dynamically allocated buffer if that
1307 * isn't large enough */
1309 s3_read_block_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
1311 s3_read_block_data *dat = stream;
1312 guint new_bytes, bytes_needed;
1314 /* if data is NULL, call through to s3_buffer_write_func */
1316 return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1319 new_bytes = (guint) size * nmemb;
1320 bytes_needed = dat->size_written + new_bytes;
1322 if (bytes_needed > (guint)dat->size_written) {
1323 /* this read will overflow the user's buffer, so malloc ourselves
1324 * a new buffer and keep reading */
1325 dat->curl.buffer = g_malloc(bytes_needed);
1326 dat->curl.buffer_len = bytes_needed;
1327 dat->curl.buffer_pos = dat->size_written;
1328 memcpy(dat->curl.buffer, dat->data, dat->size_written);
1329 dat->data = NULL; /* signal that the user's buffer is too small */
1330 return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1333 memcpy(dat->data + dat->size_written, ptr, bytes_needed);
1338 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1339 S3Device * self = S3_DEVICE(pself);
1341 s3_read_block_data dat = {NULL, 0, 0, { NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE} };
1344 g_assert (self != NULL);
1345 if (device_in_error(self)) return -1;
1348 key = file_and_block_to_key(self, pself->file, pself->block);
1349 g_assert(key != NULL);
1350 if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1351 if (*size_req >= self->cached_size) {
1352 /* use the cached copy and clear the cache */
1353 memcpy(data, self->cached_buf, self->cached_size);
1354 *size_req = self->cached_size;
1357 g_free(self->cached_key);
1358 self->cached_key = NULL;
1359 g_free(self->cached_buf);
1360 self->cached_buf = NULL;
1365 *size_req = self->cached_size;
1371 /* clear the cache, as it's useless to us */
1372 if (self->cached_key) {
1373 g_free(self->cached_key);
1374 self->cached_key = NULL;
1376 g_free(self->cached_buf);
1377 self->cached_buf = NULL;
1380 /* set up dat for the write_func callback */
1381 if (!data || *size_req <= 0) {
1386 dat.size_req = *size_req;
1389 result = s3_read(self->s3, self->bucket, key, s3_read_block_write_func,
1390 s3_buffer_reset_func, &dat, NULL, NULL);
1392 guint response_code;
1393 s3_error_code_t s3_error_code;
1394 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1399 /* if it's an expected error (not found), just return -1 */
1400 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1401 pself->is_eof = TRUE;
1402 pself->in_file = FALSE;
1403 device_set_error(pself,
1405 DEVICE_STATUS_SUCCESS);
1409 /* otherwise, log it and return FALSE */
1410 device_set_error(pself,
1411 vstrallocf(_("While reading data block from S3: %s"), s3_strerror(self->s3)),
1412 DEVICE_STATUS_VOLUME_ERROR);
1416 if (dat.data == NULL) {
1417 /* data was larger than the available space, so cache it and return
1418 * the actual size */
1419 self->cached_buf = dat.curl.buffer;
1420 self->cached_size = dat.curl.buffer_pos;
1421 self->cached_key = key;
1424 *size_req = dat.curl.buffer_pos;
1428 /* ok, the read went directly to the user's buffer, so we need only
1429 * set and return the size */
1432 *size_req = dat.size_req;
1433 return dat.size_req;