2 * Copyright (c) 2008, 2009, 2010 Zmanda, Inc. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published
6 * by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
18 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
21 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
22 * data. It stores data in keys named with a user-specified prefix, inside a
23 * user-specified bucket. Data is stored in the form of numbered (large)
29 #include <sys/types.h>
39 #include <curl/curl.h>
40 #ifdef HAVE_OPENSSL_HMAC_H
41 # include <openssl/hmac.h>
43 # ifdef HAVE_CRYPTO_HMAC_H
44 # include <crypto/hmac.h>
53 * Type checking and casting macros
55 #define TYPE_S3_DEVICE (s3_device_get_type())
56 #define S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
57 #define S3_DEVICE_CONST(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
58 #define S3_DEVICE_CLASS(klass) G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
59 #define IS_S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
61 #define S3_DEVICE_GET_CLASS(obj) G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
62 static GType s3_device_get_type (void);
65 * Main object structure
67 typedef struct _S3MetadataFile S3MetadataFile;
69 typedef struct _S3Device S3Device;
73 /* The "easy" curl handle we use to access Amazon S3 */
76 /* S3 access information */
80 /* The S3 access information. */
85 char *bucket_location;
90 /* a cache for unsuccessful reads (where we get the file but the caller
91 * doesn't have space for it or doesn't want it), where we expect the
92 * next call will request the same file.
98 /* Produce verbose output? */
105 guint64 max_send_speed;
106 guint64 max_recv_speed;
112 typedef struct _S3DeviceClass S3DeviceClass;
113 struct _S3DeviceClass {
114 DeviceClass __parent__;
119 * Constants and static data
122 #define S3_DEVICE_NAME "s3"
124 /* Maximum key length as specified in the S3 documentation
125 * (*excluding* null terminator) */
126 #define S3_MAX_KEY_LENGTH 1024
128 /* Note: for compatability, min can only be decreased and max increased */
129 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
130 #define S3_DEVICE_MAX_BLOCK_SIZE (100*1024*1024)
131 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
133 /* This goes in lieu of file number for metadata. */
134 #define SPECIAL_INFIX "special-"
136 /* pointer to the class of our parent */
137 static DeviceClass *parent_class = NULL;
140 * device-specific properties
143 /* Authentication information for Amazon S3. Both of these are strings. */
144 static DevicePropertyBase device_property_s3_access_key;
145 static DevicePropertyBase device_property_s3_secret_key;
146 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
147 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
149 /* Same, but for S3 with DevPay. */
150 static DevicePropertyBase device_property_s3_user_token;
151 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
153 /* Location constraint for new buckets created on Amazon S3. */
154 static DevicePropertyBase device_property_s3_bucket_location;
155 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
158 static DevicePropertyBase device_property_s3_storage_class;
159 #define PROPERTY_S3_STORAGE_CLASS (device_property_s3_storage_class.ID)
161 /* Path to certificate authority certificate */
162 static DevicePropertyBase device_property_ssl_ca_info;
163 #define PROPERTY_SSL_CA_INFO (device_property_ssl_ca_info.ID)
165 /* Whether to use SSL with Amazon S3. */
166 static DevicePropertyBase device_property_s3_ssl;
167 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
169 /* Speed limits for sending and receiving */
170 static DevicePropertyBase device_property_max_send_speed;
171 static DevicePropertyBase device_property_max_recv_speed;
172 #define PROPERTY_MAX_SEND_SPEED (device_property_max_send_speed.ID)
173 #define PROPERTY_MAX_RECV_SPEED (device_property_max_recv_speed.ID)
180 void s3_device_register(void);
183 * utility functions */
185 /* Given file and block numbers, return an S3 key.
187 * @param self: the S3Device object
188 * @param file: the file number
189 * @param block: the block within that file
190 * @returns: a newly allocated string containing an S3 key.
193 file_and_block_to_key(S3Device *self,
197 /* Given the name of a special file (such as 'tapestart'), generate
198 * the S3 key to use for that file.
200 * @param self: the S3Device object
201 * @param special_name: name of the special file
202 * @param file: a file number to include; omitted if -1
203 * @returns: a newly alocated string containing an S3 key.
206 special_file_to_key(S3Device *self,
209 /* Write an amanda header file to S3.
211 * @param self: the S3Device object
212 * @param label: the volume label
213 * @param timestamp: the volume timestamp
216 write_amanda_header(S3Device *self,
220 /* "Fast forward" this device to the end by looking up the largest file number
221 * present and setting the current file number one greater.
223 * @param self: the S3Device object
226 seek_to_end(S3Device *self);
228 /* Find the number of the last file that contains any data (even just a header).
230 * @param self: the S3Device object
231 * @returns: the last file, or -1 in event of an error
234 find_last_file(S3Device *self);
236 /* Delete all blocks in the given file, including the filestart block
238 * @param self: the S3Device object
239 * @param file: the file to delete
242 delete_file(S3Device *self,
246 /* Delete all files in the given device
248 * @param self: the S3Device object
251 delete_all_files(S3Device *self);
253 /* Set up self->s3 as best as possible.
255 * The return value is TRUE iff self->s3 is useable.
257 * @param self: the S3Device object
258 * @returns: TRUE if the handle is set up
261 setup_handle(S3Device * self);
267 s3_device_init(S3Device * o);
270 s3_device_class_init(S3DeviceClass * c);
273 s3_device_finalize(GObject * o);
276 s3_device_factory(char * device_name, char * device_type, char * device_node);
279 * Property{Get,Set}Fns */
281 static gboolean s3_device_set_access_key_fn(Device *self,
282 DevicePropertyBase *base, GValue *val,
283 PropertySurety surety, PropertySource source);
285 static gboolean s3_device_set_secret_key_fn(Device *self,
286 DevicePropertyBase *base, GValue *val,
287 PropertySurety surety, PropertySource source);
289 static gboolean s3_device_set_user_token_fn(Device *self,
290 DevicePropertyBase *base, GValue *val,
291 PropertySurety surety, PropertySource source);
293 static gboolean s3_device_set_bucket_location_fn(Device *self,
294 DevicePropertyBase *base, GValue *val,
295 PropertySurety surety, PropertySource source);
297 static gboolean s3_device_set_storage_class_fn(Device *self,
298 DevicePropertyBase *base, GValue *val,
299 PropertySurety surety, PropertySource source);
301 static gboolean s3_device_set_ca_info_fn(Device *self,
302 DevicePropertyBase *base, GValue *val,
303 PropertySurety surety, PropertySource source);
305 static gboolean s3_device_set_verbose_fn(Device *self,
306 DevicePropertyBase *base, GValue *val,
307 PropertySurety surety, PropertySource source);
309 static gboolean s3_device_set_ssl_fn(Device *self,
310 DevicePropertyBase *base, GValue *val,
311 PropertySurety surety, PropertySource source);
313 static gboolean s3_device_set_max_send_speed_fn(Device *self,
314 DevicePropertyBase *base, GValue *val,
315 PropertySurety surety, PropertySource source);
317 static gboolean s3_device_set_max_recv_speed_fn(Device *self,
318 DevicePropertyBase *base, GValue *val,
319 PropertySurety surety, PropertySource source);
322 * virtual functions */
325 s3_device_open_device(Device *pself, char *device_name,
326 char * device_type, char * device_node);
328 static DeviceStatusFlags s3_device_read_label(Device * self);
331 s3_device_start(Device * self,
332 DeviceAccessMode mode,
337 s3_device_finish(Device * self);
340 s3_device_start_file(Device * self,
341 dumpfile_t * jobInfo);
344 s3_device_write_block(Device * self,
349 s3_device_finish_file(Device * self);
352 s3_device_seek_file(Device *pself,
356 s3_device_seek_block(Device *pself,
360 s3_device_read_block(Device * pself,
365 s3_device_recycle_file(Device *pself,
369 s3_device_erase(Device *pself);
376 file_and_block_to_key(S3Device *self,
380 char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
381 self->prefix, file, (long long unsigned int)block);
382 g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
387 special_file_to_key(S3Device *self,
392 return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
394 return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
398 write_amanda_header(S3Device *self,
402 CurlBuffer amanda_header = {NULL, 0, 0, 0};
405 dumpfile_t * dumpinfo = NULL;
406 Device *d_self = DEVICE(self);
409 /* build the header */
410 header_size = 0; /* no minimum size */
411 dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
412 amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
414 if (amanda_header.buffer == NULL) {
415 device_set_error(d_self,
416 stralloc(_("Amanda tapestart header won't fit in a single block!")),
417 DEVICE_STATUS_DEVICE_ERROR);
418 dumpfile_free(dumpinfo);
419 g_free(amanda_header.buffer);
423 /* write out the header and flush the uploads. */
424 key = special_file_to_key(self, "tapestart", -1);
425 g_assert(header_size < G_MAXUINT); /* for cast to guint */
426 amanda_header.buffer_len = (guint)header_size;
427 result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
428 &amanda_header, NULL, NULL);
429 g_free(amanda_header.buffer);
433 device_set_error(d_self,
434 vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3)),
435 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
436 dumpfile_free(dumpinfo);
438 dumpfile_free(d_self->volume_header);
439 d_self->volume_header = dumpinfo;
446 seek_to_end(S3Device *self) {
449 Device *pself = DEVICE(self);
451 last_file = find_last_file(self);
455 pself->file = last_file;
460 /* Convert an object name into a file number, assuming the given prefix
461 * length. Returns -1 if the object name is invalid, or 0 if the object name
462 * is a "special" key. */
463 static int key_to_file(guint prefix_len, const char * key) {
467 /* skip the prefix */
468 if (strlen(key) <= prefix_len)
473 if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
477 /* check that key starts with 'f' */
482 /* check that key is of the form "%08x-" */
483 for (i = 0; i < 8; i++) {
484 if (!(key[i] >= '0' && key[i] <= '9') &&
485 !(key[i] >= 'a' && key[i] <= 'f') &&
486 !(key[i] >= 'A' && key[i] <= 'F')) break;
488 if (key[i] != '-') return -1;
489 if (i < 8) return -1;
491 /* convert the file number */
493 file = strtoul(key, NULL, 16);
495 g_warning(_("unparseable file number '%s'"), key);
502 /* Find the number of the last file that contains any data (even just a header).
503 * Returns -1 in event of an error
506 find_last_file(S3Device *self) {
509 unsigned int prefix_len = strlen(self->prefix);
511 Device *d_self = DEVICE(self);
513 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
514 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
516 device_set_error(d_self,
517 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
518 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
522 for (; keys; keys = g_slist_remove(keys, keys->data)) {
523 int file = key_to_file(prefix_len, keys->data);
525 /* and if it's the last, keep it */
526 if (file > last_file)
533 /* Find the number of the file following the requested one, if any.
534 * Returns 0 if there is no such file or -1 in event of an error
537 find_next_file(S3Device *self, int last_file) {
540 unsigned int prefix_len = strlen(self->prefix);
542 Device *d_self = DEVICE(self);
544 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
545 result = s3_list_keys(self->s3, self->bucket, self->prefix, "-", &keys);
547 device_set_error(d_self,
548 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
549 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
553 for (; keys; keys = g_slist_remove(keys, keys->data)) {
556 file = key_to_file(prefix_len, (char*)keys->data);
559 /* Set this in case we don't find a next file; this is not a
560 * hard error, so if we can find a next file we'll return that
565 if (file < next_file && file > last_file) {
574 delete_file(S3Device *self,
579 char *my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
580 Device *d_self = DEVICE(self);
582 result = s3_list_keys(self->s3, self->bucket, my_prefix, NULL, &keys);
584 device_set_error(d_self,
585 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3)),
586 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
590 /* this will likely be a *lot* of keys */
591 for (; keys; keys = g_slist_remove(keys, keys->data)) {
592 if (self->verbose) g_debug(_("Deleting %s"), (char*)keys->data);
593 if (!s3_delete(self->s3, self->bucket, keys->data)) {
594 device_set_error(d_self,
595 vstrallocf(_("While deleting key '%s': %s"),
596 (char*)keys->data, s3_strerror(self->s3)),
597 DEVICE_STATUS_DEVICE_ERROR);
607 delete_all_files(S3Device *self)
612 * Note: this has to be allowed to retry for a while because the bucket
613 * may have been created and not yet appeared
615 last_file = find_last_file(self);
618 s3_error_code_t s3_error_code;
619 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
622 * if the bucket doesn't exist, it doesn't conatin any files,
623 * so the operation is a success
625 if ((response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket)) {
626 /* find_last_file set an error; clear it */
627 device_set_error(DEVICE(self), NULL, DEVICE_STATUS_SUCCESS);
630 /* find_last_file already set the error */
635 for (file = 1; file <= last_file; file++) {
636 if (!delete_file(self, file))
637 /* delete_file already set our error message */
649 s3_device_register(void)
651 static const char * device_prefix_list[] = { S3_DEVICE_NAME, NULL };
654 /* set up our properties */
655 device_property_fill_and_register(&device_property_s3_secret_key,
656 G_TYPE_STRING, "s3_secret_key",
657 "Secret access key to authenticate with Amazon S3");
658 device_property_fill_and_register(&device_property_s3_access_key,
659 G_TYPE_STRING, "s3_access_key",
660 "Access key ID to authenticate with Amazon S3");
661 device_property_fill_and_register(&device_property_s3_user_token,
662 G_TYPE_STRING, "s3_user_token",
663 "User token for authentication Amazon devpay requests");
664 device_property_fill_and_register(&device_property_s3_bucket_location,
665 G_TYPE_STRING, "s3_bucket_location",
666 "Location constraint for buckets on Amazon S3");
667 device_property_fill_and_register(&device_property_s3_storage_class,
668 G_TYPE_STRING, "s3_storage_class",
669 "Storage class as specified by Amazon (STANDARD or REDUCED_REDUNDANCY)");
670 device_property_fill_and_register(&device_property_ssl_ca_info,
671 G_TYPE_STRING, "ssl_ca_info",
672 "Path to certificate authority certificate");
673 device_property_fill_and_register(&device_property_s3_ssl,
674 G_TYPE_BOOLEAN, "s3_ssl",
675 "Whether to use SSL with Amazon S3");
676 device_property_fill_and_register(&device_property_max_send_speed,
677 G_TYPE_UINT64, "max_send_speed",
678 "Maximum average upload speed (bytes/sec)");
679 device_property_fill_and_register(&device_property_max_recv_speed,
680 G_TYPE_UINT64, "max_recv_speed",
681 "Maximum average download speed (bytes/sec)");
683 /* register the device itself */
684 register_device(s3_device_factory, device_prefix_list);
688 s3_device_get_type(void)
690 static GType type = 0;
692 if G_UNLIKELY(type == 0) {
693 static const GTypeInfo info = {
694 sizeof (S3DeviceClass),
695 (GBaseInitFunc) NULL,
696 (GBaseFinalizeFunc) NULL,
697 (GClassInitFunc) s3_device_class_init,
698 (GClassFinalizeFunc) NULL,
699 NULL /* class_data */,
702 (GInstanceInitFunc) s3_device_init,
706 type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
714 s3_device_init(S3Device * self)
716 Device * dself = DEVICE(self);
719 /* Register property values
720 * Note: Some aren't added until s3_device_open_device()
722 bzero(&response, sizeof(response));
724 g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
725 g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
726 device_set_simple_property(dself, PROPERTY_CONCURRENCY,
727 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
728 g_value_unset(&response);
730 g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
731 g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
732 device_set_simple_property(dself, PROPERTY_STREAMING,
733 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
734 g_value_unset(&response);
736 g_value_init(&response, G_TYPE_BOOLEAN);
737 g_value_set_boolean(&response, TRUE);
738 device_set_simple_property(dself, PROPERTY_APPENDABLE,
739 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
740 g_value_unset(&response);
742 g_value_init(&response, G_TYPE_BOOLEAN);
743 g_value_set_boolean(&response, TRUE);
744 device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
745 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
746 g_value_unset(&response);
748 g_value_init(&response, G_TYPE_BOOLEAN);
749 g_value_set_boolean(&response, TRUE);
750 device_set_simple_property(dself, PROPERTY_FULL_DELETION,
751 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
752 g_value_unset(&response);
754 g_value_init(&response, G_TYPE_BOOLEAN);
755 g_value_set_boolean(&response, TRUE); /* well, there *is* no EOM on S3 .. */
756 device_set_simple_property(dself, PROPERTY_LEOM,
757 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
758 g_value_unset(&response);
760 g_value_init(&response, G_TYPE_BOOLEAN);
761 g_value_set_boolean(&response, FALSE);
762 device_set_simple_property(dself, PROPERTY_COMPRESSION,
763 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
764 g_value_unset(&response);
766 g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
767 g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
768 device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
769 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
770 g_value_unset(&response);
775 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
777 GObjectClass *g_object_class = (GObjectClass*) c;
778 DeviceClass *device_class = (DeviceClass *)c;
780 parent_class = g_type_class_ref (TYPE_DEVICE);
782 device_class->open_device = s3_device_open_device;
783 device_class->read_label = s3_device_read_label;
784 device_class->start = s3_device_start;
785 device_class->finish = s3_device_finish;
787 device_class->start_file = s3_device_start_file;
788 device_class->write_block = s3_device_write_block;
789 device_class->finish_file = s3_device_finish_file;
791 device_class->seek_file = s3_device_seek_file;
792 device_class->seek_block = s3_device_seek_block;
793 device_class->read_block = s3_device_read_block;
794 device_class->recycle_file = s3_device_recycle_file;
796 device_class->erase = s3_device_erase;
798 g_object_class->finalize = s3_device_finalize;
800 device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
801 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
802 device_simple_property_get_fn,
803 s3_device_set_access_key_fn);
805 device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
806 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
807 device_simple_property_get_fn,
808 s3_device_set_secret_key_fn);
810 device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
811 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
812 device_simple_property_get_fn,
813 s3_device_set_user_token_fn);
815 device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
816 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
817 device_simple_property_get_fn,
818 s3_device_set_bucket_location_fn);
820 device_class_register_property(device_class, PROPERTY_S3_STORAGE_CLASS,
821 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
822 device_simple_property_get_fn,
823 s3_device_set_storage_class_fn);
825 device_class_register_property(device_class, PROPERTY_SSL_CA_INFO,
826 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
827 device_simple_property_get_fn,
828 s3_device_set_ca_info_fn);
830 device_class_register_property(device_class, PROPERTY_VERBOSE,
831 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
832 device_simple_property_get_fn,
833 s3_device_set_verbose_fn);
835 device_class_register_property(device_class, PROPERTY_S3_SSL,
836 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
837 device_simple_property_get_fn,
838 s3_device_set_ssl_fn);
840 device_class_register_property(device_class, PROPERTY_MAX_SEND_SPEED,
841 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
842 device_simple_property_get_fn,
843 s3_device_set_max_send_speed_fn);
845 device_class_register_property(device_class, PROPERTY_MAX_RECV_SPEED,
846 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
847 device_simple_property_get_fn,
848 s3_device_set_max_recv_speed_fn);
850 device_class_register_property(device_class, PROPERTY_COMPRESSION,
851 PROPERTY_ACCESS_GET_MASK,
852 device_simple_property_get_fn,
857 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
858 GValue *val, PropertySurety surety, PropertySource source)
860 S3Device *self = S3_DEVICE(p_self);
862 amfree(self->access_key);
863 self->access_key = g_value_dup_string(val);
864 device_clear_volume_details(p_self);
866 return device_simple_property_set_fn(p_self, base, val, surety, source);
870 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
871 GValue *val, PropertySurety surety, PropertySource source)
873 S3Device *self = S3_DEVICE(p_self);
875 amfree(self->secret_key);
876 self->secret_key = g_value_dup_string(val);
877 device_clear_volume_details(p_self);
879 return device_simple_property_set_fn(p_self, base, val, surety, source);
883 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
884 GValue *val, PropertySurety surety, PropertySource source)
886 S3Device *self = S3_DEVICE(p_self);
888 amfree(self->user_token);
889 self->user_token = g_value_dup_string(val);
890 device_clear_volume_details(p_self);
892 return device_simple_property_set_fn(p_self, base, val, surety, source);
896 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
897 GValue *val, PropertySurety surety, PropertySource source)
899 S3Device *self = S3_DEVICE(p_self);
900 char *str_val = g_value_dup_string(val);
902 if (str_val[0] && self->use_ssl && !s3_curl_location_compat()) {
903 device_set_error(p_self, stralloc(_(
904 "Location constraint given for Amazon S3 bucket, "
905 "but libcurl is too old support wildcard certificates.")),
906 DEVICE_STATUS_DEVICE_ERROR);
910 if (str_val[0] && !s3_bucket_location_compat(self->bucket)) {
911 device_set_error(p_self, g_strdup_printf(_(
912 "Location constraint given for Amazon S3 bucket, "
913 "but the bucket name (%s) is not usable as a subdomain."),
915 DEVICE_STATUS_DEVICE_ERROR);
919 amfree(self->bucket_location);
920 self->bucket_location = str_val;
921 device_clear_volume_details(p_self);
923 return device_simple_property_set_fn(p_self, base, val, surety, source);
930 s3_device_set_storage_class_fn(Device *p_self, DevicePropertyBase *base,
931 GValue *val, PropertySurety surety, PropertySource source)
933 S3Device *self = S3_DEVICE(p_self);
934 char *str_val = g_value_dup_string(val);
936 amfree(self->storage_class);
937 self->storage_class = str_val;
938 device_clear_volume_details(p_self);
940 return device_simple_property_set_fn(p_self, base, val, surety, source);
944 s3_device_set_ca_info_fn(Device *p_self, DevicePropertyBase *base,
945 GValue *val, PropertySurety surety, PropertySource source)
947 S3Device *self = S3_DEVICE(p_self);
949 amfree(self->ca_info);
950 self->ca_info = g_value_dup_string(val);
951 device_clear_volume_details(p_self);
953 return device_simple_property_set_fn(p_self, base, val, surety, source);
957 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
958 GValue *val, PropertySurety surety, PropertySource source)
960 S3Device *self = S3_DEVICE(p_self);
962 self->verbose = g_value_get_boolean(val);
963 /* Our S3 handle may not yet have been instantiated; if so, it will
964 * get the proper verbose setting when it is created */
966 s3_verbose(self->s3, self->verbose);
968 return device_simple_property_set_fn(p_self, base, val, surety, source);
972 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
973 GValue *val, PropertySurety surety, PropertySource source)
975 S3Device *self = S3_DEVICE(p_self);
978 new_val = g_value_get_boolean(val);
979 /* Our S3 handle may not yet have been instantiated; if so, it will
980 * get the proper use_ssl setting when it is created */
981 if (self->s3 && !s3_use_ssl(self->s3, new_val)) {
982 device_set_error(p_self, g_strdup_printf(_(
983 "Error setting S3 SSL/TLS use "
984 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
985 DEVICE_STATUS_DEVICE_ERROR);
988 self->use_ssl = new_val;
990 return device_simple_property_set_fn(p_self, base, val, surety, source);
994 s3_device_set_max_send_speed_fn(Device *p_self,
995 DevicePropertyBase *base, GValue *val,
996 PropertySurety surety, PropertySource source)
998 S3Device *self = S3_DEVICE(p_self);
1001 new_val = g_value_get_uint64(val);
1002 if (self->s3 && !s3_set_max_send_speed(self->s3, new_val)) {
1003 device_set_error(p_self,
1004 g_strdup("Could not set S3 maximum send speed"),
1005 DEVICE_STATUS_DEVICE_ERROR);
1008 self->max_send_speed = new_val;
1010 return device_simple_property_set_fn(p_self, base, val, surety, source);
1014 s3_device_set_max_recv_speed_fn(Device *p_self,
1015 DevicePropertyBase *base, GValue *val,
1016 PropertySurety surety, PropertySource source)
1018 S3Device *self = S3_DEVICE(p_self);
1021 new_val = g_value_get_uint64(val);
1022 if (self->s3 && !s3_set_max_recv_speed(self->s3, new_val)) {
1023 device_set_error(p_self,
1024 g_strdup("Could not set S3 maximum recv speed"),
1025 DEVICE_STATUS_DEVICE_ERROR);
1028 self->max_recv_speed = new_val;
1030 return device_simple_property_set_fn(p_self, base, val, surety, source);
1034 s3_device_factory(char * device_name, char * device_type, char * device_node)
1038 g_assert(0 == strcmp(device_type, S3_DEVICE_NAME));
1039 rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
1040 s3_rval = (S3Device*)rval;
1042 device_open_device(rval, device_name, device_type, device_node);
1047 * Virtual function overrides
1051 s3_device_open_device(Device *pself, char *device_name,
1052 char * device_type, char * device_node)
1054 S3Device *self = S3_DEVICE(pself);
1058 pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
1059 pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
1060 pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
1062 /* Device name may be bucket/prefix, to support multiple volumes in a
1064 name_colon = strchr(device_node, '/');
1065 if (name_colon == NULL) {
1066 self->bucket = g_strdup(device_node);
1067 self->prefix = g_strdup("");
1069 self->bucket = g_strndup(device_node, name_colon - device_node);
1070 self->prefix = g_strdup(name_colon + 1);
1073 if (self->bucket == NULL || self->bucket[0] == '\0') {
1074 device_set_error(pself,
1075 vstrallocf(_("Empty bucket name in device %s"), device_name),
1076 DEVICE_STATUS_DEVICE_ERROR);
1077 amfree(self->bucket);
1078 amfree(self->prefix);
1082 g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
1084 /* default values */
1085 self->verbose = FALSE;
1087 /* use SSL if available */
1088 self->use_ssl = s3_curl_supports_ssl();
1089 bzero(&tmp_value, sizeof(GValue));
1090 g_value_init(&tmp_value, G_TYPE_BOOLEAN);
1091 g_value_set_boolean(&tmp_value, self->use_ssl);
1092 device_set_simple_property(pself, device_property_s3_ssl.ID,
1093 &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
1095 if (parent_class->open_device) {
1096 parent_class->open_device(pself, device_name, device_type, device_node);
1100 static void s3_device_finalize(GObject * obj_self) {
1101 S3Device *self = S3_DEVICE (obj_self);
1103 if(G_OBJECT_CLASS(parent_class)->finalize)
1104 (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
1106 if(self->s3) s3_free(self->s3);
1107 if(self->bucket) g_free(self->bucket);
1108 if(self->prefix) g_free(self->prefix);
1109 if(self->access_key) g_free(self->access_key);
1110 if(self->secret_key) g_free(self->secret_key);
1111 if(self->user_token) g_free(self->user_token);
1112 if(self->bucket_location) g_free(self->bucket_location);
1113 if(self->storage_class) g_free(self->storage_class);
1114 if(self->ca_info) g_free(self->ca_info);
1117 static gboolean setup_handle(S3Device * self) {
1118 Device *d_self = DEVICE(self);
1119 if (self->s3 == NULL) {
1121 if (self->access_key == NULL || self->access_key[0] == '\0') {
1122 device_set_error(d_self,
1123 stralloc(_("No Amazon access key specified")),
1124 DEVICE_STATUS_DEVICE_ERROR);
1128 if (self->secret_key == NULL || self->secret_key[0] == '\0') {
1129 device_set_error(d_self,
1130 stralloc(_("No Amazon secret key specified")),
1131 DEVICE_STATUS_DEVICE_ERROR);
1135 if (!self->use_ssl && self->ca_info) {
1136 amfree(self->ca_info);
1139 self->s3 = s3_open(self->access_key, self->secret_key, self->user_token,
1140 self->bucket_location, self->storage_class, self->ca_info);
1141 if (self->s3 == NULL) {
1142 device_set_error(d_self,
1143 stralloc(_("Internal error creating S3 handle")),
1144 DEVICE_STATUS_DEVICE_ERROR);
1149 s3_verbose(self->s3, self->verbose);
1151 if (!s3_use_ssl(self->s3, self->use_ssl)) {
1152 device_set_error(d_self, g_strdup_printf(_(
1153 "Error setting S3 SSL/TLS use "
1154 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
1155 DEVICE_STATUS_DEVICE_ERROR);
1159 if (self->max_send_speed &&
1160 !s3_set_max_send_speed(self->s3, self->max_send_speed)) {
1161 device_set_error(d_self,
1162 g_strdup("Could not set S3 maximum send speed"),
1163 DEVICE_STATUS_DEVICE_ERROR);
1167 if (self->max_recv_speed &&
1168 !s3_set_max_recv_speed(self->s3, self->max_recv_speed)) {
1169 device_set_error(d_self,
1170 g_strdup("Could not set S3 maximum recv speed"),
1171 DEVICE_STATUS_DEVICE_ERROR);
1178 static DeviceStatusFlags
1179 s3_device_read_label(Device *pself) {
1180 S3Device *self = S3_DEVICE(pself);
1182 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1183 dumpfile_t *amanda_header;
1185 /* note that this may be called from s3_device_start, when
1186 * self->access_mode is not ACCESS_NULL */
1188 amfree(pself->volume_label);
1189 amfree(pself->volume_time);
1190 dumpfile_free(pself->volume_header);
1191 pself->volume_header = NULL;
1193 if (device_in_error(self)) return pself->status;
1195 if (!setup_handle(self)) {
1196 /* setup_handle already set our error message */
1197 return pself->status;
1200 key = special_file_to_key(self, "tapestart", -1);
1201 if (!s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
1202 guint response_code;
1203 s3_error_code_t s3_error_code;
1204 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1206 /* if it's an expected error (not found), just return FALSE */
1207 if (response_code == 404 &&
1208 (s3_error_code == S3_ERROR_NoSuchKey || s3_error_code == S3_ERROR_NoSuchBucket)) {
1209 g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
1210 device_set_error(pself,
1211 stralloc(_("Amanda header not found -- unlabeled volume?")),
1212 DEVICE_STATUS_DEVICE_ERROR
1213 | DEVICE_STATUS_VOLUME_ERROR
1214 | DEVICE_STATUS_VOLUME_UNLABELED);
1215 return pself->status;
1218 /* otherwise, log it and return */
1219 device_set_error(pself,
1220 vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3)),
1221 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1222 return pself->status;
1225 /* handle an empty file gracefully */
1226 if (buf.buffer_len == 0) {
1227 device_set_error(pself, stralloc(_("Empty header file")), DEVICE_STATUS_VOLUME_ERROR);
1228 return pself->status;
1231 g_assert(buf.buffer != NULL);
1232 amanda_header = g_new(dumpfile_t, 1);
1233 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1234 pself->volume_header = amanda_header;
1237 if (amanda_header->type != F_TAPESTART) {
1238 device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
1239 return pself->status;
1242 pself->volume_label = g_strdup(amanda_header->name);
1243 pself->volume_time = g_strdup(amanda_header->datestamp);
1244 /* pself->volume_header is already set */
1246 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1248 return pself->status;
1252 s3_device_start (Device * pself, DeviceAccessMode mode,
1253 char * label, char * timestamp) {
1256 self = S3_DEVICE(pself);
1258 if (device_in_error(self)) return FALSE;
1260 if (!setup_handle(self)) {
1261 /* setup_handle already set our error message */
1265 pself->access_mode = mode;
1266 pself->in_file = FALSE;
1268 /* try creating the bucket, in case it doesn't exist */
1269 if (mode != ACCESS_READ && !s3_make_bucket(self->s3, self->bucket)) {
1270 guint response_code;
1271 s3_error_code_t s3_error_code;
1272 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1274 /* if it isn't an expected error (bucket already exists),
1276 if (response_code != 409 ||
1277 s3_error_code != S3_ERROR_BucketAlreadyExists) {
1278 device_set_error(pself,
1279 vstrallocf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3)),
1280 DEVICE_STATUS_DEVICE_ERROR);
1285 /* take care of any dirty work for this mode */
1288 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1289 /* s3_device_read_label already set our error message */
1295 delete_all_files(self);
1297 /* write a new amanda header */
1298 if (!write_amanda_header(self, label, timestamp)) {
1302 pself->volume_label = newstralloc(pself->volume_label, label);
1303 pself->volume_time = newstralloc(pself->volume_time, timestamp);
1305 /* unset the VOLUME_UNLABELED flag, if it was set */
1306 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
1310 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
1311 /* s3_device_read_label already set our error message */
1314 return seek_to_end(self);
1318 g_assert_not_reached();
1325 s3_device_finish (Device * pself) {
1326 /* we're not in a file anymore */
1327 pself->access_mode = ACCESS_NULL;
1329 if (device_in_error(pself)) return FALSE;
1334 /* functions for writing */
1338 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
1339 S3Device *self = S3_DEVICE(pself);
1340 CurlBuffer amanda_header = {NULL, 0, 0, 0};
1345 if (device_in_error(self)) return FALSE;
1347 pself->is_eom = FALSE;
1349 /* Set the blocksize to zero, since there's no header to skip (it's stored
1350 * in a distinct file, rather than block zero) */
1351 jobInfo->blocksize = 0;
1353 /* Build the amanda header. */
1354 header_size = 0; /* no minimum size */
1355 amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
1357 if (amanda_header.buffer == NULL) {
1358 device_set_error(pself,
1359 stralloc(_("Amanda file header won't fit in a single block!")),
1360 DEVICE_STATUS_DEVICE_ERROR);
1363 amanda_header.buffer_len = header_size;
1365 /* set the file and block numbers correctly */
1366 pself->file = (pself->file > 0)? pself->file+1 : 1;
1368 pself->in_file = TRUE;
1370 /* write it out as a special block (not the 0th) */
1371 key = special_file_to_key(self, "filestart", pself->file);
1372 result = s3_upload(self->s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
1373 &amanda_header, NULL, NULL);
1374 g_free(amanda_header.buffer);
1377 device_set_error(pself,
1378 vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3)),
1379 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1387 s3_device_write_block (Device * pself, guint size, gpointer data) {
1390 S3Device * self = S3_DEVICE(pself);
1391 CurlBuffer to_write = {data, size, 0, 0};
1393 g_assert (self != NULL);
1394 g_assert (data != NULL);
1395 if (device_in_error(self)) return FALSE;
1397 filename = file_and_block_to_key(self, pself->file, pself->block);
1399 result = s3_upload(self->s3, self->bucket, filename, S3_BUFFER_READ_FUNCS,
1400 &to_write, NULL, NULL);
1403 device_set_error(pself,
1404 vstrallocf(_("While writing data block to S3: %s"), s3_strerror(self->s3)),
1405 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
1415 s3_device_finish_file (Device * pself) {
1416 if (device_in_error(pself)) return FALSE;
1418 /* we're not in a file anymore */
1419 pself->in_file = FALSE;
1425 s3_device_recycle_file(Device *pself, guint file) {
1426 S3Device *self = S3_DEVICE(pself);
1427 if (device_in_error(self)) return FALSE;
1429 return delete_file(self, file);
1430 /* delete_file already set our error message if necessary */
1434 s3_device_erase(Device *pself) {
1435 S3Device *self = S3_DEVICE(pself);
1437 const char *errmsg = NULL;
1438 guint response_code;
1439 s3_error_code_t s3_error_code;
1441 if (!setup_handle(self)) {
1442 /* error set by setup_handle */
1446 key = special_file_to_key(self, "tapestart", -1);
1447 if (!s3_delete(self->s3, self->bucket, key)) {
1448 s3_error(self->s3, &errmsg, NULL, NULL, NULL, NULL, NULL);
1449 device_set_error(pself,
1451 DEVICE_STATUS_DEVICE_ERROR);
1456 if (!delete_all_files(self))
1459 if (!s3_delete_bucket(self->s3, self->bucket)) {
1460 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1463 * ignore the error if the bucket isn't empty (there may be data from elsewhere)
1464 * or the bucket not existing (already deleted perhaps?)
1467 (response_code == 409 && s3_error_code == S3_ERROR_BucketNotEmpty) ||
1468 (response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket))) {
1470 device_set_error(pself,
1472 DEVICE_STATUS_DEVICE_ERROR);
1479 /* functions for reading */
1482 s3_device_seek_file(Device *pself, guint file) {
1483 S3Device *self = S3_DEVICE(pself);
1486 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
1487 dumpfile_t *amanda_header;
1488 const char *errmsg = NULL;
1490 if (device_in_error(self)) return NULL;
1493 pself->is_eof = FALSE;
1494 pself->in_file = FALSE;
1498 key = special_file_to_key(self, "filestart", pself->file);
1499 result = s3_read(self->s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
1504 guint response_code;
1505 s3_error_code_t s3_error_code;
1506 s3_error(self->s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
1508 /* if it's an expected error (not found), check what to do. */
1509 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1511 next_file = find_next_file(self, pself->file);
1512 if (next_file > 0) {
1513 /* Note short-circut of dispatcher. */
1514 return s3_device_seek_file(pself, next_file);
1515 } else if (next_file == 0) {
1516 /* No next file. Check if we are one past the end. */
1517 key = special_file_to_key(self, "filestart", pself->file - 1);
1518 result = s3_read(self->s3, self->bucket, key,
1519 S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
1522 /* pself->file, etc. are already correct */
1523 return make_tapeend_header();
1525 device_set_error(pself,
1526 stralloc(_("Attempt to read past tape-end file")),
1527 DEVICE_STATUS_SUCCESS);
1532 /* An unexpected error occured finding out if we are the last file. */
1533 device_set_error(pself,
1535 DEVICE_STATUS_DEVICE_ERROR);
1540 /* and make a dumpfile_t out of it */
1541 g_assert(buf.buffer != NULL);
1542 amanda_header = g_new(dumpfile_t, 1);
1543 fh_init(amanda_header);
1544 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
1547 switch (amanda_header->type) {
1549 case F_CONT_DUMPFILE:
1550 case F_SPLIT_DUMPFILE:
1554 device_set_error(pself,
1555 stralloc(_("Invalid amanda header while reading file header")),
1556 DEVICE_STATUS_VOLUME_ERROR);
1557 g_free(amanda_header);
1561 pself->in_file = TRUE;
1562 return amanda_header;
1566 s3_device_seek_block(Device *pself, guint64 block) {
1567 if (device_in_error(pself)) return FALSE;
1569 pself->block = block;
1573 typedef struct s3_read_block_data {
1579 } s3_read_block_data;
1581 /* wrapper around s3_buffer_write_func to write as much data as possible to
1582 * the user's buffer, and switch to a dynamically allocated buffer if that
1583 * isn't large enough */
1585 s3_read_block_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
1587 s3_read_block_data *dat = stream;
1588 guint new_bytes, bytes_needed;
1590 /* if data is NULL, call through to s3_buffer_write_func */
1592 return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1595 new_bytes = (guint) size * nmemb;
1596 bytes_needed = dat->size_written + new_bytes;
1598 if (bytes_needed > (guint)dat->size_req) {
1599 /* this read will overflow the user's buffer, so malloc ourselves
1600 * a new buffer and keep reading */
1601 dat->curl.buffer = g_malloc(bytes_needed);
1602 dat->curl.buffer_len = bytes_needed;
1603 dat->curl.buffer_pos = dat->size_written;
1604 memcpy(dat->curl.buffer, dat->data, dat->size_written);
1605 dat->data = NULL; /* signal that the user's buffer is too small */
1606 return s3_buffer_write_func(ptr, size, nmemb, (void *)(&dat->curl));
1609 /* copy it into the dat->data buffer, and increment the size */
1610 memcpy(dat->data + dat->size_written, ptr, new_bytes);
1611 dat->size_written += new_bytes;
1617 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
1618 S3Device * self = S3_DEVICE(pself);
1620 s3_read_block_data dat = {NULL, 0, 0, { NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE} };
1623 g_assert (self != NULL);
1624 if (device_in_error(self)) return -1;
1627 key = file_and_block_to_key(self, pself->file, pself->block);
1628 g_assert(key != NULL);
1629 if (self->cached_key && (0 == strcmp(key, self->cached_key))) {
1630 if (*size_req >= self->cached_size) {
1631 /* use the cached copy and clear the cache */
1632 memcpy(data, self->cached_buf, self->cached_size);
1633 *size_req = self->cached_size;
1636 g_free(self->cached_key);
1637 self->cached_key = NULL;
1638 g_free(self->cached_buf);
1639 self->cached_buf = NULL;
1644 *size_req = self->cached_size;
1650 /* clear the cache, as it's useless to us */
1651 if (self->cached_key) {
1652 g_free(self->cached_key);
1653 self->cached_key = NULL;
1655 g_free(self->cached_buf);
1656 self->cached_buf = NULL;
1659 /* set up dat for the write_func callback */
1660 if (!data || *size_req <= 0) {
1665 dat.size_req = *size_req;
1668 result = s3_read(self->s3, self->bucket, key, s3_read_block_write_func,
1669 s3_buffer_reset_func, &dat, NULL, NULL);
1671 guint response_code;
1672 s3_error_code_t s3_error_code;
1673 s3_error(self->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
1678 /* if it's an expected error (not found), just return -1 */
1679 if (response_code == 404 && s3_error_code == S3_ERROR_NoSuchKey) {
1680 pself->is_eof = TRUE;
1681 pself->in_file = FALSE;
1682 device_set_error(pself,
1684 DEVICE_STATUS_SUCCESS);
1688 /* otherwise, log it and return FALSE */
1689 device_set_error(pself,
1690 vstrallocf(_("While reading data block from S3: %s"), s3_strerror(self->s3)),
1691 DEVICE_STATUS_VOLUME_ERROR);
1695 if (dat.data == NULL) {
1696 /* data was larger than the available space, so cache it and return
1697 * the actual size */
1698 self->cached_buf = dat.curl.buffer;
1699 self->cached_size = dat.curl.buffer_pos;
1700 self->cached_key = key;
1703 *size_req = dat.curl.buffer_pos;
1707 /* ok, the read went directly to the user's buffer, so we need only
1708 * set and return the size */
1711 *size_req = dat.size_written;
1712 return dat.size_written;