2 * Copyright (c) 2008-2012 Zmanda, Inc. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version 2
7 * of the License, or (at your option) any later version.
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * You should have received a copy of the GNU General Public License along
15 * with this program; if not, write to the Free Software Foundation, Inc.,
16 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
19 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
22 /* An S3 device uses Amazon's S3 service (http://www.amazon.com/s3) to store
23 * data. It stores data in keys named with a user-specified prefix, inside a
24 * user-specified bucket. Data is stored in the form of numbered (large)
30 #include <sys/types.h>
40 #include <curl/curl.h>
41 #ifdef HAVE_OPENSSL_HMAC_H
42 # include <openssl/hmac.h>
44 # ifdef HAVE_CRYPTO_HMAC_H
45 # include <crypto/hmac.h>
54 * Type checking and casting macros
56 #define TYPE_S3_DEVICE (s3_device_get_type())
57 #define S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device)
58 #define S3_DEVICE_CONST(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), s3_device_get_type(), S3Device const)
59 #define S3_DEVICE_CLASS(klass) G_TYPE_CHECK_CLASS_CAST((klass), s3_device_get_type(), S3DeviceClass)
60 #define IS_S3_DEVICE(obj) G_TYPE_CHECK_INSTANCE_TYPE((obj), s3_device_get_type ())
62 #define S3_DEVICE_GET_CLASS(obj) G_TYPE_INSTANCE_GET_CLASS((obj), s3_device_get_type(), S3DeviceClass)
63 static GType s3_device_get_type (void);
66 * Main object structure
68 typedef struct _S3MetadataFile S3MetadataFile;
69 typedef struct _S3Device S3Device;
71 typedef struct _S3_by_thread S3_by_thread;
72 struct _S3_by_thread {
73 S3Handle * volatile s3;
74 CurlBuffer volatile curl_buffer;
75 guint volatile buffer_len;
79 char volatile * volatile filename;
80 DeviceStatusFlags volatile errflags; /* device_status */
81 char volatile * volatile errmsg; /* device error message */
89 /* The "easy" curl handle we use to access Amazon S3 */
92 /* S3 access information */
96 /* The S3 access information. */
101 /* The Openstack swift information. */
102 char *swift_account_id;
103 char *swift_access_key;
110 char *bucket_location;
114 char *server_side_encryption;
119 /* a cache for unsuccessful reads (where we get the file but the caller
120 * doesn't have space for it or doesn't want it), where we expect the
121 * next call will request the same file.
127 /* Produce verbose output? */
130 /* create the bucket? */
131 gboolean create_bucket;
138 guint64 max_send_speed;
139 guint64 max_recv_speed;
142 guint64 volume_bytes;
143 guint64 volume_limit;
144 gboolean enforce_volume_limit;
145 gboolean use_subdomain;
146 gboolean use_s3_multi_delete;
149 int nb_threads_backup;
150 int nb_threads_recovery;
151 GThreadPool *thread_pool_delete;
152 GThreadPool *thread_pool_write;
153 GThreadPool *thread_pool_read;
154 GCond *thread_idle_cond;
155 GMutex *thread_idle_mutex;
156 int next_block_to_read;
168 gboolean reuse_connection;
178 typedef struct _S3DeviceClass S3DeviceClass;
179 struct _S3DeviceClass {
180 DeviceClass __parent__;
185 * Constants and static data
188 #define S3_DEVICE_NAME "s3"
190 /* Maximum key length as specified in the S3 documentation
191 * (*excluding* null terminator) */
192 #define S3_MAX_KEY_LENGTH 1024
194 /* Note: for compatability, min can only be decreased and max increased */
195 #define S3_DEVICE_MIN_BLOCK_SIZE 1024
196 #define S3_DEVICE_MAX_BLOCK_SIZE (3*1024*1024*1024ULL)
197 #define S3_DEVICE_DEFAULT_BLOCK_SIZE (10*1024*1024)
198 #define EOM_EARLY_WARNING_ZONE_BLOCKS 4
200 /* This goes in lieu of file number for metadata. */
201 #define SPECIAL_INFIX "special-"
203 /* pointer to the class of our parent */
204 static DeviceClass *parent_class = NULL;
207 * device-specific properties
210 /* Authentication information for Amazon S3. Both of these are strings. */
211 static DevicePropertyBase device_property_s3_access_key;
212 static DevicePropertyBase device_property_s3_secret_key;
213 #define PROPERTY_S3_SECRET_KEY (device_property_s3_secret_key.ID)
214 #define PROPERTY_S3_ACCESS_KEY (device_property_s3_access_key.ID)
216 /* Authentication information for Openstack Swift. Both of these are strings. */
217 static DevicePropertyBase device_property_swift_account_id;
218 static DevicePropertyBase device_property_swift_access_key;
219 #define PROPERTY_SWIFT_ACCOUNT_ID (device_property_swift_account_id.ID)
220 #define PROPERTY_SWIFT_ACCESS_KEY (device_property_swift_access_key.ID)
222 /* Authentication information for Openstack Swift. Both of these are strings. */
223 static DevicePropertyBase device_property_username;
224 static DevicePropertyBase device_property_password;
225 static DevicePropertyBase device_property_tenant_id;
226 static DevicePropertyBase device_property_tenant_name;
227 #define PROPERTY_USERNAME (device_property_username.ID)
228 #define PROPERTY_PASSWORD (device_property_password.ID)
229 #define PROPERTY_TENANT_ID (device_property_tenant_id.ID)
230 #define PROPERTY_TENANT_NAME (device_property_tenant_name.ID)
233 static DevicePropertyBase device_property_s3_host;
234 static DevicePropertyBase device_property_s3_service_path;
235 #define PROPERTY_S3_HOST (device_property_s3_host.ID)
236 #define PROPERTY_S3_SERVICE_PATH (device_property_s3_service_path.ID)
238 /* Same, but for S3 with DevPay. */
239 static DevicePropertyBase device_property_s3_user_token;
240 #define PROPERTY_S3_USER_TOKEN (device_property_s3_user_token.ID)
242 /* Location constraint for new buckets created on Amazon S3. */
243 static DevicePropertyBase device_property_s3_bucket_location;
244 #define PROPERTY_S3_BUCKET_LOCATION (device_property_s3_bucket_location.ID)
247 static DevicePropertyBase device_property_s3_storage_class;
248 #define PROPERTY_S3_STORAGE_CLASS (device_property_s3_storage_class.ID)
250 /* Server side encryption */
251 static DevicePropertyBase device_property_s3_server_side_encryption;
252 #define PROPERTY_S3_SERVER_SIDE_ENCRYPTION (device_property_s3_server_side_encryption.ID)
255 static DevicePropertyBase device_property_proxy;
256 #define PROPERTY_PROXY (device_property_proxy.ID)
258 /* Path to certificate authority certificate */
259 static DevicePropertyBase device_property_ssl_ca_info;
260 #define PROPERTY_SSL_CA_INFO (device_property_ssl_ca_info.ID)
262 /* Which strotage api to use. */
263 static DevicePropertyBase device_property_storage_api;
264 #define PROPERTY_STORAGE_API (device_property_storage_api.ID)
266 /* Whether to use openstack protocol. */
268 static DevicePropertyBase device_property_openstack_swift_api;
269 #define PROPERTY_OPENSTACK_SWIFT_API (device_property_openstack_swift_api.ID)
271 /* Whether to use SSL with Amazon S3. */
272 static DevicePropertyBase device_property_s3_ssl;
273 #define PROPERTY_S3_SSL (device_property_s3_ssl.ID)
275 /* Whether to re-use connection. */
276 static DevicePropertyBase device_property_reuse_connection;
277 #define PROPERTY_REUSE_CONNECTION (device_property_reuse_connection.ID)
279 /* Speed limits for sending and receiving */
280 static DevicePropertyBase device_property_max_send_speed;
281 static DevicePropertyBase device_property_max_recv_speed;
282 #define PROPERTY_MAX_SEND_SPEED (device_property_max_send_speed.ID)
283 #define PROPERTY_MAX_RECV_SPEED (device_property_max_recv_speed.ID)
285 /* Whether to use subdomain */
286 static DevicePropertyBase device_property_s3_subdomain;
287 #define PROPERTY_S3_SUBDOMAIN (device_property_s3_subdomain.ID)
289 /* Number of threads to use */
290 static DevicePropertyBase device_property_nb_threads_backup;
291 #define PROPERTY_NB_THREADS_BACKUP (device_property_nb_threads_backup.ID)
292 static DevicePropertyBase device_property_nb_threads_recovery;
293 #define PROPERTY_NB_THREADS_RECOVERY (device_property_nb_threads_recovery.ID)
295 /* If the s3 server have the multi-delete functionality */
296 static DevicePropertyBase device_property_s3_multi_delete;
297 #define PROPERTY_S3_MULTI_DELETE (device_property_s3_multi_delete.ID)
299 /* The client_id for OAUTH2 */
300 static DevicePropertyBase device_property_client_id;
301 #define PROPERTY_CLIENT_ID (device_property_client_id.ID)
303 /* The client_secret for OAUTH2 */
304 static DevicePropertyBase device_property_client_secret;
305 #define PROPERTY_CLIENT_SECRET (device_property_client_secret.ID)
307 /* The refresh token for OAUTH2 */
308 static DevicePropertyBase device_property_refresh_token;
309 #define PROPERTY_REFRESH_TOKEN (device_property_refresh_token.ID)
312 static DevicePropertyBase device_property_project_id;
313 #define PROPERTY_PROJECT_ID (device_property_project_id.ID)
316 static DevicePropertyBase device_property_create_bucket;
317 #define PROPERTY_CREATE_BUCKET (device_property_create_bucket.ID)
319 /* CAStor replication values for objects and buckets */
320 static DevicePropertyBase device_property_s3_reps;
321 #define PROPERTY_S3_REPS (device_property_s3_reps.ID)
322 #define S3_DEVICE_REPS_DEFAULT "2"
323 static DevicePropertyBase device_property_s3_reps_bucket;
324 #define PROPERTY_S3_REPS_BUCKET (device_property_s3_reps_bucket.ID)
325 #define S3_DEVICE_REPS_BUCKET_DEFAULT "4"
331 void s3_device_register(void);
334 * utility functions */
336 /* Given file and block numbers, return an S3 key.
338 * @param self: the S3Device object
339 * @param file: the file number
340 * @param block: the block within that file
341 * @returns: a newly allocated string containing an S3 key.
344 file_and_block_to_key(S3Device *self,
348 /* Given the name of a special file (such as 'tapestart'), generate
349 * the S3 key to use for that file.
351 * @param self: the S3Device object
352 * @param special_name: name of the special file
353 * @param file: a file number to include; omitted if -1
354 * @returns: a newly alocated string containing an S3 key.
357 special_file_to_key(S3Device *self,
360 /* Write an amanda header file to S3.
362 * @param self: the S3Device object
363 * @param label: the volume label
364 * @param timestamp: the volume timestamp
367 write_amanda_header(S3Device *self,
371 /* "Fast forward" this device to the end by looking up the largest file number
372 * present and setting the current file number one greater.
374 * @param self: the S3Device object
377 seek_to_end(S3Device *self);
379 /* Find the number of the last file that contains any data (even just a header).
381 * @param self: the S3Device object
382 * @returns: the last file, or -1 in event of an error
385 find_last_file(S3Device *self);
387 /* Delete all blocks in the given file, including the filestart block
389 * @param self: the S3Device object
390 * @param file: the file to delete
393 delete_file(S3Device *self,
397 /* Delete all files in the given device
399 * @param self: the S3Device object
402 delete_all_files(S3Device *self);
404 /* Set up self->s3t as best as possible.
406 * The return value is TRUE iff self->s3t is useable.
408 * @param self: the S3Device object
409 * @returns: TRUE if the handle is set up
412 setup_handle(S3Device * self);
415 s3_wait_thread_delete(S3Device *self);
421 s3_device_init(S3Device * o);
424 s3_device_class_init(S3DeviceClass * c);
427 s3_device_finalize(GObject * o);
430 s3_device_factory(char * device_name, char * device_type, char * device_node);
433 * Property{Get,Set}Fns */
435 static gboolean s3_device_set_access_key_fn(Device *self,
436 DevicePropertyBase *base, GValue *val,
437 PropertySurety surety, PropertySource source);
439 static gboolean s3_device_set_secret_key_fn(Device *self,
440 DevicePropertyBase *base, GValue *val,
441 PropertySurety surety, PropertySource source);
443 static gboolean s3_device_set_swift_account_id_fn(Device *self,
444 DevicePropertyBase *base, GValue *val,
445 PropertySurety surety, PropertySource source);
447 static gboolean s3_device_set_swift_access_key_fn(Device *self,
448 DevicePropertyBase *base, GValue *val,
449 PropertySurety surety, PropertySource source);
451 static gboolean s3_device_set_username(Device *self,
452 DevicePropertyBase *base, GValue *val,
453 PropertySurety surety, PropertySource source);
455 static gboolean s3_device_set_password(Device *self,
456 DevicePropertyBase *base, GValue *val,
457 PropertySurety surety, PropertySource source);
459 static gboolean s3_device_set_tenant_id(Device *self,
460 DevicePropertyBase *base, GValue *val,
461 PropertySurety surety, PropertySource source);
463 static gboolean s3_device_set_tenant_name(Device *self,
464 DevicePropertyBase *base, GValue *val,
465 PropertySurety surety, PropertySource source);
467 static gboolean s3_device_set_user_token_fn(Device *self,
468 DevicePropertyBase *base, GValue *val,
469 PropertySurety surety, PropertySource source);
471 static gboolean s3_device_set_bucket_location_fn(Device *self,
472 DevicePropertyBase *base, GValue *val,
473 PropertySurety surety, PropertySource source);
475 static gboolean s3_device_set_storage_class_fn(Device *self,
476 DevicePropertyBase *base, GValue *val,
477 PropertySurety surety, PropertySource source);
479 static gboolean s3_device_set_server_side_encryption_fn(Device *self,
480 DevicePropertyBase *base, GValue *val,
481 PropertySurety surety, PropertySource source);
483 static gboolean s3_device_set_proxy_fn(Device *self,
484 DevicePropertyBase *base, GValue *val,
485 PropertySurety surety, PropertySource source);
487 static gboolean s3_device_set_ca_info_fn(Device *self,
488 DevicePropertyBase *base, GValue *val,
489 PropertySurety surety, PropertySource source);
491 static gboolean s3_device_set_verbose_fn(Device *self,
492 DevicePropertyBase *base, GValue *val,
493 PropertySurety surety, PropertySource source);
495 static gboolean s3_device_set_create_bucket_fn(Device *self,
496 DevicePropertyBase *base, GValue *val,
497 PropertySurety surety, PropertySource source);
499 static gboolean s3_device_set_storage_api(Device *self,
500 DevicePropertyBase *base, GValue *val,
501 PropertySurety surety, PropertySource source);
503 static gboolean s3_device_set_openstack_swift_api_fn(Device *self,
504 DevicePropertyBase *base, GValue *val,
505 PropertySurety surety, PropertySource source);
507 static gboolean s3_device_set_s3_multi_delete_fn(Device *self,
508 DevicePropertyBase *base, GValue *val,
509 PropertySurety surety, PropertySource source);
511 static gboolean s3_device_set_ssl_fn(Device *self,
512 DevicePropertyBase *base, GValue *val,
513 PropertySurety surety, PropertySource source);
515 static gboolean s3_device_set_reuse_connection_fn(Device *self,
516 DevicePropertyBase *base, GValue *val,
517 PropertySurety surety, PropertySource source);
519 static gboolean s3_device_set_max_send_speed_fn(Device *self,
520 DevicePropertyBase *base, GValue *val,
521 PropertySurety surety, PropertySource source);
523 static gboolean s3_device_set_max_recv_speed_fn(Device *self,
524 DevicePropertyBase *base, GValue *val,
525 PropertySurety surety, PropertySource source);
527 static gboolean s3_device_set_nb_threads_backup(Device *self,
528 DevicePropertyBase *base, GValue *val,
529 PropertySurety surety, PropertySource source);
531 static gboolean s3_device_set_nb_threads_recovery(Device *self,
532 DevicePropertyBase *base, GValue *val,
533 PropertySurety surety, PropertySource source);
535 static gboolean s3_device_set_max_volume_usage_fn(Device *p_self,
536 DevicePropertyBase *base, GValue *val,
537 PropertySurety surety, PropertySource source);
539 static gboolean property_set_leom_fn(Device *p_self,
540 DevicePropertyBase *base, GValue *val,
541 PropertySurety surety, PropertySource source);
543 static gboolean s3_device_set_enforce_max_volume_usage_fn(Device *p_self,
544 DevicePropertyBase *base, GValue *val,
545 PropertySurety surety, PropertySource source);
547 static gboolean s3_device_set_use_subdomain_fn(Device *p_self,
548 DevicePropertyBase *base, GValue *val,
549 PropertySurety surety, PropertySource source);
551 static gboolean s3_device_set_host_fn(Device *p_self,
552 DevicePropertyBase *base, GValue *val,
553 PropertySurety surety, PropertySource source);
555 static gboolean s3_device_set_service_path_fn(Device *p_self,
556 DevicePropertyBase *base, GValue *val,
557 PropertySurety surety, PropertySource source);
559 static gboolean s3_device_set_client_id_fn(Device *p_self,
560 DevicePropertyBase *base, GValue *val,
561 PropertySurety surety, PropertySource source);
563 static gboolean s3_device_set_client_secret_fn(Device *p_self,
564 DevicePropertyBase *base, GValue *val,
565 PropertySurety surety, PropertySource source);
567 static gboolean s3_device_set_refresh_token_fn(Device *p_self,
568 DevicePropertyBase *base, GValue *val,
569 PropertySurety surety, PropertySource source);
571 static gboolean s3_device_set_project_id_fn(Device *p_self,
572 DevicePropertyBase *base, GValue *val,
573 PropertySurety surety, PropertySource source);
575 static gboolean s3_device_set_reps_fn(Device *self,
576 DevicePropertyBase *base, GValue *val,
577 PropertySurety surety, PropertySource source);
579 static gboolean s3_device_set_reps_bucket_fn(Device *self,
580 DevicePropertyBase *base, GValue *val,
581 PropertySurety surety, PropertySource source);
583 static void s3_thread_read_block(gpointer thread_data,
585 static void s3_thread_write_block(gpointer thread_data,
587 static gboolean make_bucket(Device * pself);
590 /* Wait that all threads are done */
591 static void reset_thread(S3Device *self);
594 * virtual functions */
597 s3_device_open_device(Device *pself, char *device_name,
598 char * device_type, char * device_node);
600 static DeviceStatusFlags s3_device_read_label(Device * self);
603 s3_device_start(Device * self,
604 DeviceAccessMode mode,
609 s3_device_finish(Device * self);
612 s3_device_get_bytes_read(Device * self);
615 s3_device_get_bytes_written(Device * self);
618 s3_device_start_file(Device * self,
619 dumpfile_t * jobInfo);
622 s3_device_write_block(Device * self,
627 s3_device_finish_file(Device * self);
630 s3_device_seek_file(Device *pself,
634 s3_device_seek_block(Device *pself,
638 s3_device_read_block(Device * pself,
643 s3_device_recycle_file(Device *pself,
647 s3_device_erase(Device *pself);
650 check_at_leom(S3Device *self,
654 check_at_peom(S3Device *self,
662 file_and_block_to_key(S3Device *self,
666 char *s3_key = g_strdup_printf("%sf%08x-b%016llx.data",
667 self->prefix, file, (long long unsigned int)block);
668 g_assert(strlen(s3_key) <= S3_MAX_KEY_LENGTH);
673 special_file_to_key(S3Device *self,
678 return g_strdup_printf("%s" SPECIAL_INFIX "%s", self->prefix, special_name);
680 return g_strdup_printf("%sf%08x-%s", self->prefix, file, special_name);
684 write_amanda_header(S3Device *self,
688 CurlBuffer amanda_header = {NULL, 0, 0, 0};
691 dumpfile_t * dumpinfo = NULL;
692 Device *d_self = DEVICE(self);
695 /* build the header */
696 header_size = 0; /* no minimum size */
697 dumpinfo = make_tapestart_header(DEVICE(self), label, timestamp);
698 amanda_header.buffer = device_build_amanda_header(DEVICE(self), dumpinfo,
700 if (amanda_header.buffer == NULL) {
701 device_set_error(d_self,
702 stralloc(_("Amanda tapestart header won't fit in a single block!")),
703 DEVICE_STATUS_DEVICE_ERROR);
704 dumpfile_free(dumpinfo);
705 g_free(amanda_header.buffer);
709 if(check_at_leom(self, header_size))
710 d_self->is_eom = TRUE;
712 if(check_at_peom(self, header_size)) {
713 d_self->is_eom = TRUE;
714 device_set_error(d_self,
715 stralloc(_("No space left on device")),
716 DEVICE_STATUS_DEVICE_ERROR);
717 g_free(amanda_header.buffer);
721 /* write out the header and flush the uploads. */
722 key = special_file_to_key(self, "tapestart", -1);
723 g_assert(header_size < G_MAXUINT); /* for cast to guint */
724 amanda_header.buffer_len = (guint)header_size;
725 result = s3_upload(self->s3t[0].s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
726 &amanda_header, NULL, NULL);
727 g_free(amanda_header.buffer);
731 device_set_error(d_self,
732 vstrallocf(_("While writing amanda header: %s"), s3_strerror(self->s3t[0].s3)),
733 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
734 dumpfile_free(dumpinfo);
736 dumpfile_free(d_self->volume_header);
737 d_self->volume_header = dumpinfo;
738 self->volume_bytes += header_size;
740 d_self->header_block_size = header_size;
745 seek_to_end(S3Device *self) {
748 Device *pself = DEVICE(self);
750 last_file = find_last_file(self);
754 pself->file = last_file;
759 /* Convert an object name into a file number, assuming the given prefix
760 * length. Returns -1 if the object name is invalid, or 0 if the object name
761 * is a "special" key. */
762 static int key_to_file(guint prefix_len, const char * key) {
766 /* skip the prefix */
767 if (strlen(key) <= prefix_len)
772 if (strncmp(key, SPECIAL_INFIX, strlen(SPECIAL_INFIX)) == 0) {
776 /* check that key starts with 'f' */
781 /* check that key is of the form "%08x-" */
782 for (i = 0; i < 8; i++) {
783 if (!(key[i] >= '0' && key[i] <= '9') &&
784 !(key[i] >= 'a' && key[i] <= 'f') &&
785 !(key[i] >= 'A' && key[i] <= 'F')) break;
787 if (key[i] != '-') return -1;
788 if (i < 8) return -1;
790 /* convert the file number */
792 file = strtoul(key, NULL, 16);
794 g_warning(_("unparseable file number '%s'"), key);
801 /* Find the number of the last file that contains any data (even just a header).
802 * Returns -1 in event of an error
805 find_last_file(S3Device *self) {
808 unsigned int prefix_len = strlen(self->prefix);
810 Device *d_self = DEVICE(self);
812 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
813 result = s3_list_keys(self->s3t[0].s3, self->bucket, self->prefix, "-", &keys, NULL);
815 device_set_error(d_self,
816 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3t[0].s3)),
817 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
821 for (; keys; keys = g_slist_remove(keys, keys->data)) {
822 int file = key_to_file(prefix_len, keys->data);
824 /* and if it's the last, keep it */
825 if (file > last_file)
832 /* Find the number of the file following the requested one, if any.
833 * Returns 0 if there is no such file or -1 in event of an error
836 find_next_file(S3Device *self, int last_file) {
839 unsigned int prefix_len = strlen(self->prefix);
841 Device *d_self = DEVICE(self);
843 /* list all keys matching C{PREFIX*-*}, stripping the C{-*} */
844 result = s3_list_keys(self->s3t[0].s3, self->bucket, self->prefix, "-",
847 device_set_error(d_self,
848 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3t[0].s3)),
849 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
853 for (; keys; keys = g_slist_remove(keys, keys->data)) {
856 file = key_to_file(prefix_len, (char*)keys->data);
859 /* Set this in case we don't find a next file; this is not a
860 * hard error, so if we can find a next file we'll return that
865 if (file < next_file && file > last_file) {
874 delete_file(S3Device *self,
881 guint64 total_size = 0;
882 Device *d_self = DEVICE(self);
886 my_prefix = g_strdup_printf("%sf", self->prefix);
888 my_prefix = g_strdup_printf("%sf%08x-", self->prefix, file);
891 result = s3_list_keys(self->s3t[0].s3, self->bucket, my_prefix, NULL,
894 device_set_error(d_self,
895 g_strdup_printf(_("While listing S3 keys: %s"),
896 s3_strerror(self->s3t[0].s3)),
897 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
901 g_mutex_lock(self->thread_idle_mutex);
905 self->keys = g_slist_concat(self->keys, keys);
909 for (thread = 0; thread < self->nb_threads; thread++) {
910 if (self->s3t[thread].idle == 1) {
911 /* Check if the thread is in error */
912 if (self->s3t[thread].errflags != DEVICE_STATUS_SUCCESS) {
913 device_set_error(d_self,
914 (char *)self->s3t[thread].errmsg,
915 self->s3t[thread].errflags);
916 self->s3t[thread].errflags = DEVICE_STATUS_SUCCESS;
917 self->s3t[thread].errmsg = NULL;
918 g_mutex_unlock(self->thread_idle_mutex);
919 s3_wait_thread_delete(self);
922 self->s3t[thread].idle = 0;
923 self->s3t[thread].done = 0;
924 g_thread_pool_push(self->thread_pool_delete, &self->s3t[thread],
928 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
929 g_mutex_unlock(self->thread_idle_mutex);
931 self->volume_bytes = total_size;
933 s3_wait_thread_delete(self);
939 s3_thread_delete_block(
940 gpointer thread_data,
943 static int count = 0;
944 S3_by_thread *s3t = (S3_by_thread *)thread_data;
945 Device *pself = (Device *)data;
946 S3Device *self = S3_DEVICE(pself);
950 g_mutex_lock(self->thread_idle_mutex);
951 while (result && self->keys) {
952 if (self->use_s3_multi_delete) {
953 char **filenames = g_new(char *, 1001);
954 char **f = filenames;
956 while (self->keys && n<1000) {
957 *f++ = self->keys->data;
958 self->keys = g_slist_remove(self->keys, self->keys->data);
962 g_mutex_unlock(self->thread_idle_mutex);
963 result = s3_multi_delete(s3t->s3, (const char *)self->bucket,
964 (const char **)filenames);
969 g_debug("Deleting multiple keys not implemented");
970 } else { /* result == 0 */
971 g_debug("Deleteing multiple keys failed: %s",
972 s3_strerror(s3t->s3));
975 self->use_s3_multi_delete = 0;
976 /* re-add all filenames */
978 g_mutex_lock(self->thread_idle_mutex);
980 self->keys = g_slist_prepend(self->keys, *f++);
982 g_mutex_unlock(self->thread_idle_mutex);
985 g_mutex_lock(self->thread_idle_mutex);
994 filename = self->keys->data;
995 self->keys = g_slist_remove(self->keys, self->keys->data);
998 g_debug("Deleting %s ...", filename);
1001 g_mutex_unlock(self->thread_idle_mutex);
1002 result = s3_delete(s3t->s3, (const char *)self->bucket,
1003 (const char *)filename);
1005 s3t->errflags = DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR;
1006 s3t->errmsg = g_strdup_printf(_("While deleting key '%s': %s"),
1007 filename, s3_strerror(s3t->s3));
1011 g_mutex_lock(self->thread_idle_mutex);
1015 g_cond_broadcast(self->thread_idle_cond);
1016 g_mutex_unlock(self->thread_idle_mutex);
1020 s3_wait_thread_delete(S3Device *self)
1022 Device *d_self = (Device *)self;
1023 int idle_thread = 0;
1026 g_mutex_lock(self->thread_idle_mutex);
1027 while (idle_thread != self->nb_threads) {
1029 for (thread = 0; thread < self->nb_threads; thread++) {
1030 if (self->s3t[thread].idle == 1) {
1033 /* Check if the thread is in error */
1034 if (self->s3t[thread].errflags != DEVICE_STATUS_SUCCESS) {
1035 device_set_error(d_self, (char *)self->s3t[thread].errmsg,
1036 self->s3t[thread].errflags);
1037 self->s3t[thread].errflags = DEVICE_STATUS_SUCCESS;
1038 self->s3t[thread].errmsg = NULL;
1041 if (idle_thread != self->nb_threads) {
1042 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
1045 g_mutex_unlock(self->thread_idle_mutex);
1050 delete_all_files(S3Device *self)
1052 return delete_file(self, -1);
1060 s3_device_register(void)
1062 static const char * device_prefix_list[] = { S3_DEVICE_NAME, NULL };
1063 g_assert(s3_init());
1065 /* set up our properties */
1066 device_property_fill_and_register(&device_property_s3_secret_key,
1067 G_TYPE_STRING, "s3_secret_key",
1068 "Secret access key to authenticate with Amazon S3");
1069 device_property_fill_and_register(&device_property_s3_access_key,
1070 G_TYPE_STRING, "s3_access_key",
1071 "Access key ID to authenticate with Amazon S3");
1072 device_property_fill_and_register(&device_property_swift_account_id,
1073 G_TYPE_STRING, "swift_account_id",
1074 "Account ID to authenticate with openstack swift");
1075 device_property_fill_and_register(&device_property_swift_access_key,
1076 G_TYPE_STRING, "swift_access_key",
1077 "Access key to authenticate with openstack swift");
1078 device_property_fill_and_register(&device_property_username,
1079 G_TYPE_STRING, "username",
1080 "Username to authenticate with");
1081 device_property_fill_and_register(&device_property_password,
1082 G_TYPE_STRING, "password",
1083 "password to authenticate with");
1084 device_property_fill_and_register(&device_property_tenant_id,
1085 G_TYPE_STRING, "tenant_id",
1086 "tenant_id to authenticate with");
1087 device_property_fill_and_register(&device_property_tenant_name,
1088 G_TYPE_STRING, "tenant_name",
1089 "tenant_name to authenticate with");
1090 device_property_fill_and_register(&device_property_s3_host,
1091 G_TYPE_STRING, "s3_host",
1092 "hostname:port of the server");
1093 device_property_fill_and_register(&device_property_s3_service_path,
1094 G_TYPE_STRING, "s3_service_path",
1095 "path to add in the url");
1096 device_property_fill_and_register(&device_property_s3_user_token,
1097 G_TYPE_STRING, "s3_user_token",
1098 "User token for authentication Amazon devpay requests");
1099 device_property_fill_and_register(&device_property_s3_bucket_location,
1100 G_TYPE_STRING, "s3_bucket_location",
1101 "Location constraint for buckets on Amazon S3");
1102 device_property_fill_and_register(&device_property_s3_storage_class,
1103 G_TYPE_STRING, "s3_storage_class",
1104 "Storage class as specified by Amazon (STANDARD or REDUCED_REDUNDANCY)");
1105 device_property_fill_and_register(&device_property_s3_server_side_encryption,
1106 G_TYPE_STRING, "s3_server_side_encryption",
1107 "Serve side encryption as specified by Amazon (AES256)");
1108 device_property_fill_and_register(&device_property_proxy,
1109 G_TYPE_STRING, "proxy",
1111 device_property_fill_and_register(&device_property_ssl_ca_info,
1112 G_TYPE_STRING, "ssl_ca_info",
1113 "Path to certificate authority certificate");
1114 device_property_fill_and_register(&device_property_storage_api,
1115 G_TYPE_STRING, "storage_api",
1116 "Which cloud API to use.");
1117 device_property_fill_and_register(&device_property_openstack_swift_api,
1118 G_TYPE_STRING, "openstack_swift_api",
1119 "Whether to use openstack protocol");
1120 device_property_fill_and_register(&device_property_client_id,
1121 G_TYPE_STRING, "client_id",
1122 "client_id for use with oauth2");
1123 device_property_fill_and_register(&device_property_client_secret,
1124 G_TYPE_STRING, "client_secret",
1125 "client_secret for use with oauth2");
1126 device_property_fill_and_register(&device_property_refresh_token,
1127 G_TYPE_STRING, "refresh_token",
1128 "refresh_token for use with oauth2");
1129 device_property_fill_and_register(&device_property_project_id,
1130 G_TYPE_STRING, "project_id",
1131 "project id for use with google");
1132 device_property_fill_and_register(&device_property_s3_ssl,
1133 G_TYPE_BOOLEAN, "s3_ssl",
1134 "Whether to use SSL with Amazon S3");
1135 device_property_fill_and_register(&device_property_reuse_connection,
1136 G_TYPE_BOOLEAN, "reuse_connection",
1137 "Whether to reuse connection");
1138 device_property_fill_and_register(&device_property_create_bucket,
1139 G_TYPE_BOOLEAN, "create_bucket",
1140 "Whether to create/delete bucket");
1141 device_property_fill_and_register(&device_property_s3_subdomain,
1142 G_TYPE_BOOLEAN, "s3_subdomain",
1143 "Whether to use subdomain");
1144 device_property_fill_and_register(&device_property_max_send_speed,
1145 G_TYPE_UINT64, "max_send_speed",
1146 "Maximum average upload speed (bytes/sec)");
1147 device_property_fill_and_register(&device_property_max_recv_speed,
1148 G_TYPE_UINT64, "max_recv_speed",
1149 "Maximum average download speed (bytes/sec)");
1150 device_property_fill_and_register(&device_property_nb_threads_backup,
1151 G_TYPE_UINT64, "nb_threads_backup",
1152 "Number of writer thread");
1153 device_property_fill_and_register(&device_property_nb_threads_recovery,
1154 G_TYPE_UINT64, "nb_threads_recovery",
1155 "Number of reader thread");
1156 device_property_fill_and_register(&device_property_s3_multi_delete,
1157 G_TYPE_BOOLEAN, "s3_multi_delete",
1158 "Whether to use multi-delete");
1159 device_property_fill_and_register(&device_property_s3_reps,
1160 G_TYPE_STRING, "reps",
1161 "Number of replicas for data objects in CAStor");
1162 device_property_fill_and_register(&device_property_s3_reps_bucket,
1163 G_TYPE_STRING, "reps_bucket",
1164 "Number of replicas for automatically created buckets in CAStor");
1166 /* register the device itself */
1167 register_device(s3_device_factory, device_prefix_list);
1171 s3_device_get_type(void)
1173 static GType type = 0;
1175 if G_UNLIKELY(type == 0) {
1176 static const GTypeInfo info = {
1177 sizeof (S3DeviceClass),
1178 (GBaseInitFunc) NULL,
1179 (GBaseFinalizeFunc) NULL,
1180 (GClassInitFunc) s3_device_class_init,
1181 (GClassFinalizeFunc) NULL,
1182 NULL /* class_data */,
1184 0 /* n_preallocs */,
1185 (GInstanceInitFunc) s3_device_init,
1189 type = g_type_register_static (TYPE_DEVICE, "S3Device", &info,
1197 s3_device_init(S3Device * self)
1199 Device * dself = DEVICE(self);
1202 self->s3_api = S3_API_S3;
1203 self->volume_bytes = 0;
1204 self->volume_limit = 0;
1206 self->enforce_volume_limit = FALSE;
1207 self->use_subdomain = FALSE;
1208 self->nb_threads = 1;
1209 self->nb_threads_backup = 1;
1210 self->nb_threads_recovery = 1;
1211 self->thread_pool_delete = NULL;
1212 self->thread_pool_write = NULL;
1213 self->thread_pool_read = NULL;
1214 self->thread_idle_cond = NULL;
1215 self->thread_idle_mutex = NULL;
1216 self->use_s3_multi_delete = 1;
1218 self->reps_bucket = NULL;
1220 /* Register property values
1221 * Note: Some aren't added until s3_device_open_device()
1223 bzero(&response, sizeof(response));
1225 g_value_init(&response, CONCURRENCY_PARADIGM_TYPE);
1226 g_value_set_enum(&response, CONCURRENCY_PARADIGM_SHARED_READ);
1227 device_set_simple_property(dself, PROPERTY_CONCURRENCY,
1228 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1229 g_value_unset(&response);
1231 g_value_init(&response, STREAMING_REQUIREMENT_TYPE);
1232 g_value_set_enum(&response, STREAMING_REQUIREMENT_NONE);
1233 device_set_simple_property(dself, PROPERTY_STREAMING,
1234 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1235 g_value_unset(&response);
1237 g_value_init(&response, G_TYPE_BOOLEAN);
1238 g_value_set_boolean(&response, TRUE);
1239 device_set_simple_property(dself, PROPERTY_APPENDABLE,
1240 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1241 g_value_unset(&response);
1243 g_value_init(&response, G_TYPE_BOOLEAN);
1244 g_value_set_boolean(&response, TRUE);
1245 device_set_simple_property(dself, PROPERTY_PARTIAL_DELETION,
1246 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1247 g_value_unset(&response);
1249 g_value_init(&response, G_TYPE_BOOLEAN);
1250 g_value_set_boolean(&response, TRUE);
1251 device_set_simple_property(dself, PROPERTY_FULL_DELETION,
1252 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1253 g_value_unset(&response);
1255 g_value_init(&response, G_TYPE_BOOLEAN);
1256 g_value_set_boolean(&response, TRUE); /* well, there *is* no EOM on S3 .. */
1257 device_set_simple_property(dself, PROPERTY_LEOM,
1258 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1259 g_value_unset(&response);
1261 g_value_init(&response, G_TYPE_BOOLEAN);
1262 g_value_set_boolean(&response, FALSE);
1263 device_set_simple_property(dself, PROPERTY_ENFORCE_MAX_VOLUME_USAGE,
1264 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1265 g_value_unset(&response);
1267 g_value_init(&response, G_TYPE_BOOLEAN);
1268 g_value_set_boolean(&response, FALSE);
1269 device_set_simple_property(dself, PROPERTY_S3_SUBDOMAIN,
1270 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1271 g_value_unset(&response);
1273 g_value_init(&response, G_TYPE_BOOLEAN);
1274 g_value_set_boolean(&response, FALSE);
1275 device_set_simple_property(dself, PROPERTY_COMPRESSION,
1276 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1277 g_value_unset(&response);
1279 g_value_init(&response, MEDIA_ACCESS_MODE_TYPE);
1280 g_value_set_enum(&response, MEDIA_ACCESS_MODE_READ_WRITE);
1281 device_set_simple_property(dself, PROPERTY_MEDIUM_ACCESS_TYPE,
1282 &response, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DETECTED);
1283 g_value_unset(&response);
1288 s3_device_class_init(S3DeviceClass * c G_GNUC_UNUSED)
1290 GObjectClass *g_object_class = (GObjectClass*) c;
1291 DeviceClass *device_class = (DeviceClass *)c;
1293 parent_class = g_type_class_ref (TYPE_DEVICE);
1295 device_class->open_device = s3_device_open_device;
1296 device_class->read_label = s3_device_read_label;
1297 device_class->start = s3_device_start;
1298 device_class->finish = s3_device_finish;
1299 device_class->get_bytes_read = s3_device_get_bytes_read;
1300 device_class->get_bytes_written = s3_device_get_bytes_written;
1302 device_class->start_file = s3_device_start_file;
1303 device_class->write_block = s3_device_write_block;
1304 device_class->finish_file = s3_device_finish_file;
1306 device_class->seek_file = s3_device_seek_file;
1307 device_class->seek_block = s3_device_seek_block;
1308 device_class->read_block = s3_device_read_block;
1309 device_class->recycle_file = s3_device_recycle_file;
1311 device_class->erase = s3_device_erase;
1313 g_object_class->finalize = s3_device_finalize;
1315 device_class_register_property(device_class, PROPERTY_S3_ACCESS_KEY,
1316 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1317 device_simple_property_get_fn,
1318 s3_device_set_access_key_fn);
1320 device_class_register_property(device_class, PROPERTY_S3_SECRET_KEY,
1321 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1322 device_simple_property_get_fn,
1323 s3_device_set_secret_key_fn);
1325 device_class_register_property(device_class, PROPERTY_SWIFT_ACCOUNT_ID,
1326 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1327 device_simple_property_get_fn,
1328 s3_device_set_swift_account_id_fn);
1330 device_class_register_property(device_class, PROPERTY_SWIFT_ACCESS_KEY,
1331 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1332 device_simple_property_get_fn,
1333 s3_device_set_swift_access_key_fn);
1335 device_class_register_property(device_class, PROPERTY_USERNAME,
1336 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1337 device_simple_property_get_fn,
1338 s3_device_set_username);
1340 device_class_register_property(device_class, PROPERTY_PASSWORD,
1341 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1342 device_simple_property_get_fn,
1343 s3_device_set_password);
1345 device_class_register_property(device_class, PROPERTY_TENANT_ID,
1346 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1347 device_simple_property_get_fn,
1348 s3_device_set_tenant_id);
1350 device_class_register_property(device_class, PROPERTY_TENANT_NAME,
1351 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1352 device_simple_property_get_fn,
1353 s3_device_set_tenant_name);
1355 device_class_register_property(device_class, PROPERTY_S3_HOST,
1356 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1357 device_simple_property_get_fn,
1358 s3_device_set_host_fn);
1360 device_class_register_property(device_class, PROPERTY_S3_SERVICE_PATH,
1361 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1362 device_simple_property_get_fn,
1363 s3_device_set_service_path_fn);
1365 device_class_register_property(device_class, PROPERTY_S3_USER_TOKEN,
1366 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1367 device_simple_property_get_fn,
1368 s3_device_set_user_token_fn);
1370 device_class_register_property(device_class, PROPERTY_S3_BUCKET_LOCATION,
1371 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1372 device_simple_property_get_fn,
1373 s3_device_set_bucket_location_fn);
1375 device_class_register_property(device_class, PROPERTY_S3_STORAGE_CLASS,
1376 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1377 device_simple_property_get_fn,
1378 s3_device_set_storage_class_fn);
1380 device_class_register_property(device_class, PROPERTY_S3_SERVER_SIDE_ENCRYPTION,
1381 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1382 device_simple_property_get_fn,
1383 s3_device_set_server_side_encryption_fn);
1385 device_class_register_property(device_class, PROPERTY_PROXY,
1386 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1387 device_simple_property_get_fn,
1388 s3_device_set_proxy_fn);
1390 device_class_register_property(device_class, PROPERTY_SSL_CA_INFO,
1391 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1392 device_simple_property_get_fn,
1393 s3_device_set_ca_info_fn);
1395 device_class_register_property(device_class, PROPERTY_VERBOSE,
1396 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1397 device_simple_property_get_fn,
1398 s3_device_set_verbose_fn);
1400 device_class_register_property(device_class, PROPERTY_CREATE_BUCKET,
1401 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1402 device_simple_property_get_fn,
1403 s3_device_set_create_bucket_fn);
1405 device_class_register_property(device_class, PROPERTY_STORAGE_API,
1406 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1407 device_simple_property_get_fn,
1408 s3_device_set_storage_api);
1410 device_class_register_property(device_class, PROPERTY_OPENSTACK_SWIFT_API,
1411 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1412 device_simple_property_get_fn,
1413 s3_device_set_openstack_swift_api_fn);
1415 device_class_register_property(device_class, PROPERTY_S3_MULTI_DELETE,
1416 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1417 device_simple_property_get_fn,
1418 s3_device_set_s3_multi_delete_fn);
1420 device_class_register_property(device_class, PROPERTY_S3_SSL,
1421 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1422 device_simple_property_get_fn,
1423 s3_device_set_ssl_fn);
1425 device_class_register_property(device_class, PROPERTY_REUSE_CONNECTION,
1426 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1427 device_simple_property_get_fn,
1428 s3_device_set_reuse_connection_fn);
1430 device_class_register_property(device_class, PROPERTY_MAX_SEND_SPEED,
1431 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1432 device_simple_property_get_fn,
1433 s3_device_set_max_send_speed_fn);
1435 device_class_register_property(device_class, PROPERTY_MAX_RECV_SPEED,
1436 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1437 device_simple_property_get_fn,
1438 s3_device_set_max_recv_speed_fn);
1440 device_class_register_property(device_class, PROPERTY_NB_THREADS_BACKUP,
1441 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1442 device_simple_property_get_fn,
1443 s3_device_set_nb_threads_backup);
1445 device_class_register_property(device_class, PROPERTY_NB_THREADS_RECOVERY,
1446 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1447 device_simple_property_get_fn,
1448 s3_device_set_nb_threads_recovery);
1450 device_class_register_property(device_class, PROPERTY_COMPRESSION,
1451 PROPERTY_ACCESS_GET_MASK,
1452 device_simple_property_get_fn,
1455 device_class_register_property(device_class, PROPERTY_LEOM,
1456 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1457 device_simple_property_get_fn,
1458 property_set_leom_fn);
1460 device_class_register_property(device_class, PROPERTY_MAX_VOLUME_USAGE,
1461 (PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_MASK) &
1462 (~ PROPERTY_ACCESS_SET_INSIDE_FILE_WRITE),
1463 device_simple_property_get_fn,
1464 s3_device_set_max_volume_usage_fn);
1466 device_class_register_property(device_class, PROPERTY_ENFORCE_MAX_VOLUME_USAGE,
1467 (PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_MASK) &
1468 (~ PROPERTY_ACCESS_SET_INSIDE_FILE_WRITE),
1469 device_simple_property_get_fn,
1470 s3_device_set_enforce_max_volume_usage_fn);
1472 device_class_register_property(device_class, PROPERTY_S3_SUBDOMAIN,
1473 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1474 device_simple_property_get_fn,
1475 s3_device_set_use_subdomain_fn);
1477 device_class_register_property(device_class, PROPERTY_CLIENT_ID,
1478 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1479 device_simple_property_get_fn,
1480 s3_device_set_client_id_fn);
1482 device_class_register_property(device_class, PROPERTY_CLIENT_SECRET,
1483 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1484 device_simple_property_get_fn,
1485 s3_device_set_client_secret_fn);
1487 device_class_register_property(device_class, PROPERTY_REFRESH_TOKEN,
1488 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1489 device_simple_property_get_fn,
1490 s3_device_set_refresh_token_fn);
1492 device_class_register_property(device_class, PROPERTY_PROJECT_ID,
1493 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1494 device_simple_property_get_fn,
1495 s3_device_set_project_id_fn);
1497 device_class_register_property(device_class, PROPERTY_S3_REPS,
1498 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1499 device_simple_property_get_fn,
1500 s3_device_set_reps_fn);
1502 device_class_register_property(device_class, PROPERTY_S3_REPS_BUCKET,
1503 PROPERTY_ACCESS_GET_MASK | PROPERTY_ACCESS_SET_BEFORE_START,
1504 device_simple_property_get_fn,
1505 s3_device_set_reps_bucket_fn);
1509 s3_device_set_access_key_fn(Device *p_self, DevicePropertyBase *base,
1510 GValue *val, PropertySurety surety, PropertySource source)
1512 S3Device *self = S3_DEVICE(p_self);
1514 amfree(self->access_key);
1515 self->access_key = g_value_dup_string(val);
1516 device_clear_volume_details(p_self);
1518 return device_simple_property_set_fn(p_self, base, val, surety, source);
1522 s3_device_set_secret_key_fn(Device *p_self, DevicePropertyBase *base,
1523 GValue *val, PropertySurety surety, PropertySource source)
1525 S3Device *self = S3_DEVICE(p_self);
1527 amfree(self->secret_key);
1528 self->secret_key = g_value_dup_string(val);
1529 device_clear_volume_details(p_self);
1531 return device_simple_property_set_fn(p_self, base, val, surety, source);
1535 s3_device_set_swift_account_id_fn(Device *p_self, DevicePropertyBase *base,
1536 GValue *val, PropertySurety surety, PropertySource source)
1538 S3Device *self = S3_DEVICE(p_self);
1540 amfree(self->swift_account_id);
1541 self->swift_account_id = g_value_dup_string(val);
1542 device_clear_volume_details(p_self);
1544 return device_simple_property_set_fn(p_self, base, val, surety, source);
1548 s3_device_set_swift_access_key_fn(Device *p_self, DevicePropertyBase *base,
1549 GValue *val, PropertySurety surety, PropertySource source)
1551 S3Device *self = S3_DEVICE(p_self);
1553 amfree(self->swift_access_key);
1554 self->swift_access_key = g_value_dup_string(val);
1555 device_clear_volume_details(p_self);
1557 return device_simple_property_set_fn(p_self, base, val, surety, source);
1561 s3_device_set_username(Device *p_self, DevicePropertyBase *base,
1562 GValue *val, PropertySurety surety, PropertySource source)
1564 S3Device *self = S3_DEVICE(p_self);
1566 amfree(self->username);
1567 self->username = g_value_dup_string(val);
1568 device_clear_volume_details(p_self);
1570 return device_simple_property_set_fn(p_self, base, val, surety, source);
1574 s3_device_set_password(Device *p_self, DevicePropertyBase *base,
1575 GValue *val, PropertySurety surety, PropertySource source)
1577 S3Device *self = S3_DEVICE(p_self);
1579 amfree(self->password);
1580 self->password = g_value_dup_string(val);
1581 device_clear_volume_details(p_self);
1583 return device_simple_property_set_fn(p_self, base, val, surety, source);
1587 s3_device_set_tenant_id(Device *p_self, DevicePropertyBase *base,
1588 GValue *val, PropertySurety surety, PropertySource source)
1590 S3Device *self = S3_DEVICE(p_self);
1592 amfree(self->tenant_id);
1593 self->tenant_id = g_value_dup_string(val);
1594 device_clear_volume_details(p_self);
1596 return device_simple_property_set_fn(p_self, base, val, surety, source);
1600 s3_device_set_tenant_name(Device *p_self, DevicePropertyBase *base,
1601 GValue *val, PropertySurety surety, PropertySource source)
1603 S3Device *self = S3_DEVICE(p_self);
1605 amfree(self->tenant_name);
1606 self->tenant_name = g_value_dup_string(val);
1607 device_clear_volume_details(p_self);
1609 return device_simple_property_set_fn(p_self, base, val, surety, source);
1613 s3_device_set_host_fn(Device *p_self,
1614 DevicePropertyBase *base, GValue *val,
1615 PropertySurety surety, PropertySource source)
1617 S3Device *self = S3_DEVICE(p_self);
1620 self->host = g_value_dup_string(val);
1621 device_clear_volume_details(p_self);
1623 return device_simple_property_set_fn(p_self, base, val, surety, source);
1627 s3_device_set_service_path_fn(Device *p_self,
1628 DevicePropertyBase *base, GValue *val,
1629 PropertySurety surety, PropertySource source)
1631 S3Device *self = S3_DEVICE(p_self);
1633 amfree(self->service_path);
1634 self->service_path = g_value_dup_string(val);
1635 device_clear_volume_details(p_self);
1637 return device_simple_property_set_fn(p_self, base, val, surety, source);
1641 s3_device_set_user_token_fn(Device *p_self, DevicePropertyBase *base,
1642 GValue *val, PropertySurety surety, PropertySource source)
1644 S3Device *self = S3_DEVICE(p_self);
1646 amfree(self->user_token);
1647 self->user_token = g_value_dup_string(val);
1648 device_clear_volume_details(p_self);
1650 return device_simple_property_set_fn(p_self, base, val, surety, source);
1654 s3_device_set_bucket_location_fn(Device *p_self, DevicePropertyBase *base,
1655 GValue *val, PropertySurety surety, PropertySource source)
1657 S3Device *self = S3_DEVICE(p_self);
1658 char *str_val = g_value_dup_string(val);
1660 if (str_val[0] && self->use_ssl && !s3_curl_location_compat()) {
1661 device_set_error(p_self, stralloc(_(
1662 "Location constraint given for Amazon S3 bucket, "
1663 "but libcurl is too old support wildcard certificates.")),
1664 DEVICE_STATUS_DEVICE_ERROR);
1668 if (str_val[0] && !s3_bucket_location_compat(self->bucket)) {
1669 device_set_error(p_self, g_strdup_printf(_(
1670 "Location constraint given for Amazon S3 bucket, "
1671 "but the bucket name (%s) is not usable as a subdomain."),
1673 DEVICE_STATUS_DEVICE_ERROR);
1677 amfree(self->bucket_location);
1678 self->bucket_location = str_val;
1679 device_clear_volume_details(p_self);
1681 return device_simple_property_set_fn(p_self, base, val, surety, source);
1688 s3_device_set_storage_class_fn(Device *p_self, DevicePropertyBase *base,
1689 GValue *val, PropertySurety surety, PropertySource source)
1691 S3Device *self = S3_DEVICE(p_self);
1692 char *str_val = g_value_dup_string(val);
1694 amfree(self->storage_class);
1695 self->storage_class = str_val;
1696 device_clear_volume_details(p_self);
1698 return device_simple_property_set_fn(p_self, base, val, surety, source);
1702 s3_device_set_server_side_encryption_fn(Device *p_self, DevicePropertyBase *base,
1703 GValue *val, PropertySurety surety, PropertySource source)
1705 S3Device *self = S3_DEVICE(p_self);
1706 char *str_val = g_value_dup_string(val);
1708 amfree(self->server_side_encryption);
1709 self->server_side_encryption = str_val;
1710 device_clear_volume_details(p_self);
1712 return device_simple_property_set_fn(p_self, base, val, surety, source);
1716 s3_device_set_proxy_fn(Device *p_self, DevicePropertyBase *base,
1717 GValue *val, PropertySurety surety, PropertySource source)
1719 S3Device *self = S3_DEVICE(p_self);
1720 char *str_val = g_value_dup_string(val);
1722 amfree(self->proxy);
1723 self->proxy = str_val;
1724 device_clear_volume_details(p_self);
1726 return device_simple_property_set_fn(p_self, base, val, surety, source);
1730 s3_device_set_ca_info_fn(Device *p_self, DevicePropertyBase *base,
1731 GValue *val, PropertySurety surety, PropertySource source)
1733 S3Device *self = S3_DEVICE(p_self);
1735 amfree(self->ca_info);
1736 self->ca_info = g_value_dup_string(val);
1737 device_clear_volume_details(p_self);
1739 return device_simple_property_set_fn(p_self, base, val, surety, source);
1743 s3_device_set_verbose_fn(Device *p_self, DevicePropertyBase *base,
1744 GValue *val, PropertySurety surety, PropertySource source)
1746 S3Device *self = S3_DEVICE(p_self);
1749 self->verbose = g_value_get_boolean(val);
1750 /* Our S3 handle may not yet have been instantiated; if so, it will
1751 * get the proper verbose setting when it is created */
1753 for (thread = 0; thread < self->nb_threads; thread++) {
1754 if (self->s3t[thread].s3)
1755 s3_verbose(self->s3t[thread].s3, self->verbose);
1759 return device_simple_property_set_fn(p_self, base, val, surety, source);
1763 s3_device_set_create_bucket_fn(Device *p_self, DevicePropertyBase *base,
1764 GValue *val, PropertySurety surety, PropertySource source)
1766 S3Device *self = S3_DEVICE(p_self);
1769 self->create_bucket = g_value_get_boolean(val);
1770 /* Our S3 handle may not yet have been instantiated; if so, it will
1771 * get the proper verbose setting when it is created */
1773 for (thread = 0; thread < self->nb_threads; thread++) {
1774 if (self->s3t[thread].s3)
1775 s3_verbose(self->s3t[thread].s3, self->verbose);
1779 return device_simple_property_set_fn(p_self, base, val, surety, source);
1783 s3_device_set_storage_api(Device *p_self, DevicePropertyBase *base,
1784 GValue *val, PropertySurety surety, PropertySource source)
1786 S3Device *self = S3_DEVICE(p_self);
1788 const char *storage_api = g_value_get_string(val);
1789 if (g_str_equal(storage_api, "S3")) {
1790 self->s3_api = S3_API_S3;
1791 } else if (g_str_equal(storage_api, "SWIFT-1.0")) {
1792 self->s3_api = S3_API_SWIFT_1;
1793 } else if (g_str_equal(storage_api, "SWIFT-2.0")) {
1794 self->s3_api = S3_API_SWIFT_2;
1795 } else if (g_str_equal(storage_api, "OAUTH2")) {
1796 self->s3_api = S3_API_OAUTH2;
1797 } else if (g_str_equal(storage_api, "CASTOR")) {
1798 #if LIBCURL_VERSION_NUM >= 0x071301
1799 curl_version_info_data *info;
1800 /* check the runtime version too */
1801 info = curl_version_info(CURLVERSION_NOW);
1802 if (info->version_num >= 0x071301) {
1803 self->s3_api = S3_API_CASTOR;
1805 device_set_error(p_self, g_strdup_printf(_(
1806 "Error setting STORAGE-API to castor "
1807 "(You must install libcurl 7.19.1 or newer)")),
1808 DEVICE_STATUS_DEVICE_ERROR);
1812 device_set_error(p_self, g_strdup_printf(_(
1813 "Error setting STORAGE-API to castor "
1814 "This amanda is compiled with a too old libcurl, you must compile with libcurl 7.19.1 or newer")),
1815 DEVICE_STATUS_DEVICE_ERROR);
1819 g_debug("Invalid STORAGE_API, using \"S3\".");
1820 self->s3_api = S3_API_S3;
1823 return device_simple_property_set_fn(p_self, base, val, surety, source);
1827 s3_device_set_openstack_swift_api_fn(Device *p_self, DevicePropertyBase *base,
1828 GValue *val, PropertySurety surety, PropertySource source)
1831 const gboolean openstack_swift_api = g_value_get_boolean(val);
1832 if (openstack_swift_api) {
1833 GValue storage_api_val;
1834 g_value_init(&storage_api_val, G_TYPE_STRING);
1835 g_value_set_static_string(&storage_api_val, "SWIFT-1.0");
1836 return s3_device_set_storage_api(p_self, base, &storage_api_val,
1843 s3_device_set_s3_multi_delete_fn(Device *p_self,
1844 DevicePropertyBase *base, GValue *val,
1845 PropertySurety surety, PropertySource source)
1847 S3Device *self = S3_DEVICE(p_self);
1849 self->use_s3_multi_delete = g_value_get_boolean(val);
1851 return device_simple_property_set_fn(p_self, base, val, surety, source);
1855 s3_device_set_ssl_fn(Device *p_self, DevicePropertyBase *base,
1856 GValue *val, PropertySurety surety, PropertySource source)
1858 S3Device *self = S3_DEVICE(p_self);
1862 new_val = g_value_get_boolean(val);
1863 /* Our S3 handle may not yet have been instantiated; if so, it will
1864 * get the proper use_ssl setting when it is created */
1866 for (thread = 0; thread < self->nb_threads; thread++) {
1867 if (self->s3t[thread].s3 && !s3_use_ssl(self->s3t[thread].s3, new_val)) {
1868 device_set_error(p_self, g_strdup_printf(_(
1869 "Error setting S3 SSL/TLS use "
1870 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
1871 DEVICE_STATUS_DEVICE_ERROR);
1876 self->use_ssl = new_val;
1878 return device_simple_property_set_fn(p_self, base, val, surety, source);
1882 s3_device_set_reuse_connection_fn(Device *p_self, DevicePropertyBase *base,
1883 GValue *val, PropertySurety surety, PropertySource source)
1885 S3Device *self = S3_DEVICE(p_self);
1887 self->reuse_connection = g_value_get_boolean(val);
1889 return device_simple_property_set_fn(p_self, base, val, surety, source);
1893 s3_device_set_max_send_speed_fn(Device *p_self,
1894 DevicePropertyBase *base, GValue *val,
1895 PropertySurety surety, PropertySource source)
1897 S3Device *self = S3_DEVICE(p_self);
1901 new_val = g_value_get_uint64(val);
1903 for (thread = 0; thread < self->nb_threads; thread++) {
1904 if (self->s3t[thread].s3 && !s3_set_max_send_speed(self->s3t[thread].s3, new_val)) {
1905 device_set_error(p_self,
1906 g_strdup("Could not set S3 maximum send speed"),
1907 DEVICE_STATUS_DEVICE_ERROR);
1912 self->max_send_speed = new_val;
1914 return device_simple_property_set_fn(p_self, base, val, surety, source);
1918 s3_device_set_max_recv_speed_fn(Device *p_self,
1919 DevicePropertyBase *base, GValue *val,
1920 PropertySurety surety, PropertySource source)
1922 S3Device *self = S3_DEVICE(p_self);
1926 new_val = g_value_get_uint64(val);
1928 for (thread = 0; thread < self->nb_threads; thread++) {
1929 if (self->s3t[thread].s3 &&
1930 !s3_set_max_recv_speed(self->s3t[thread].s3, new_val)) {
1931 device_set_error(p_self,
1932 g_strdup("Could not set S3 maximum recv speed"),
1933 DEVICE_STATUS_DEVICE_ERROR);
1938 self->max_recv_speed = new_val;
1940 return device_simple_property_set_fn(p_self, base, val, surety, source);
1944 s3_device_set_nb_threads_backup(Device *p_self,
1945 DevicePropertyBase *base, GValue *val,
1946 PropertySurety surety, PropertySource source)
1948 S3Device *self = S3_DEVICE(p_self);
1951 new_val = g_value_get_uint64(val);
1952 self->nb_threads_backup = new_val;
1953 if (self->nb_threads_backup > self->nb_threads) {
1954 self->nb_threads = self->nb_threads_backup;
1957 return device_simple_property_set_fn(p_self, base, val, surety, source);
1961 s3_device_set_nb_threads_recovery(Device *p_self,
1962 DevicePropertyBase *base, GValue *val,
1963 PropertySurety surety, PropertySource source)
1965 S3Device *self = S3_DEVICE(p_self);
1968 new_val = g_value_get_uint64(val);
1969 self->nb_threads_recovery = new_val;
1970 if (self->nb_threads_recovery > self->nb_threads) {
1971 self->nb_threads = self->nb_threads_recovery;
1974 return device_simple_property_set_fn(p_self, base, val, surety, source);
1978 s3_device_set_max_volume_usage_fn(Device *p_self,
1979 DevicePropertyBase *base, GValue *val,
1980 PropertySurety surety, PropertySource source)
1982 S3Device *self = S3_DEVICE(p_self);
1984 self->volume_limit = g_value_get_uint64(val);
1986 return device_simple_property_set_fn(p_self, base, val, surety, source);
1991 s3_device_set_enforce_max_volume_usage_fn(Device *p_self,
1992 DevicePropertyBase *base, GValue *val,
1993 PropertySurety surety, PropertySource source)
1995 S3Device *self = S3_DEVICE(p_self);
1997 self->enforce_volume_limit = g_value_get_boolean(val);
1999 return device_simple_property_set_fn(p_self, base, val, surety, source);
2004 s3_device_set_use_subdomain_fn(Device *p_self,
2005 DevicePropertyBase *base, GValue *val,
2006 PropertySurety surety, PropertySource source)
2008 S3Device *self = S3_DEVICE(p_self);
2010 self->use_subdomain = g_value_get_boolean(val);
2012 return device_simple_property_set_fn(p_self, base, val, surety, source);
2016 property_set_leom_fn(Device *p_self,
2017 DevicePropertyBase *base, GValue *val,
2018 PropertySurety surety, PropertySource source)
2020 S3Device *self = S3_DEVICE(p_self);
2022 self->leom = g_value_get_boolean(val);
2024 return device_simple_property_set_fn(p_self, base, val, surety, source);
2028 s3_device_set_client_id_fn(Device *p_self,
2029 DevicePropertyBase *base, GValue *val,
2030 PropertySurety surety, PropertySource source)
2032 S3Device *self = S3_DEVICE(p_self);
2034 amfree(self->client_id);
2035 self->client_id = g_value_dup_string(val);
2037 return device_simple_property_set_fn(p_self, base, val, surety, source);
2041 s3_device_set_client_secret_fn(Device *p_self,
2042 DevicePropertyBase *base, GValue *val,
2043 PropertySurety surety, PropertySource source)
2045 S3Device *self = S3_DEVICE(p_self);
2047 amfree(self->client_secret);
2048 self->client_secret = g_value_dup_string(val);
2050 return device_simple_property_set_fn(p_self, base, val, surety, source);
2054 s3_device_set_refresh_token_fn(Device *p_self,
2055 DevicePropertyBase *base, GValue *val,
2056 PropertySurety surety, PropertySource source)
2058 S3Device *self = S3_DEVICE(p_self);
2060 amfree(self->refresh_token);
2061 self->refresh_token = g_value_dup_string(val);
2063 return device_simple_property_set_fn(p_self, base, val, surety, source);
2067 s3_device_set_project_id_fn(Device *p_self,
2068 DevicePropertyBase *base, GValue *val,
2069 PropertySurety surety, PropertySource source)
2071 S3Device *self = S3_DEVICE(p_self);
2073 amfree(self->project_id);
2074 self->project_id = g_value_dup_string(val);
2076 return device_simple_property_set_fn(p_self, base, val, surety, source);
2080 s3_device_set_reps_fn(Device *p_self, DevicePropertyBase *base,
2081 GValue *val, PropertySurety surety, PropertySource source)
2083 S3Device *self = S3_DEVICE(p_self);
2086 self->reps = g_value_dup_string(val);
2087 device_clear_volume_details(p_self);
2089 return device_simple_property_set_fn(p_self, base, val, surety, source);
2093 s3_device_set_reps_bucket_fn(Device *p_self, DevicePropertyBase *base,
2094 GValue *val, PropertySurety surety, PropertySource source)
2096 S3Device *self = S3_DEVICE(p_self);
2098 amfree(self->reps_bucket);
2099 self->reps_bucket = g_value_dup_string(val);
2100 device_clear_volume_details(p_self);
2102 return device_simple_property_set_fn(p_self, base, val, surety, source);
2106 s3_device_factory(char * device_name, char * device_type, char * device_node)
2109 g_assert(0 == strcmp(device_type, S3_DEVICE_NAME));
2110 rval = DEVICE(g_object_new(TYPE_S3_DEVICE, NULL));
2112 device_open_device(rval, device_name, device_type, device_node);
2117 * Virtual function overrides
2121 s3_device_open_device(Device *pself, char *device_name,
2122 char * device_type, char * device_node)
2124 S3Device *self = S3_DEVICE(pself);
2128 pself->min_block_size = S3_DEVICE_MIN_BLOCK_SIZE;
2129 pself->max_block_size = S3_DEVICE_MAX_BLOCK_SIZE;
2130 pself->block_size = S3_DEVICE_DEFAULT_BLOCK_SIZE;
2132 /* Device name may be bucket/prefix, to support multiple volumes in a
2134 name_colon = strchr(device_node, '/');
2135 if (name_colon == NULL) {
2136 self->bucket = g_strdup(device_node);
2137 self->prefix = g_strdup("");
2139 self->bucket = g_strndup(device_node, name_colon - device_node);
2140 self->prefix = g_strdup(name_colon + 1);
2143 if (self->bucket == NULL || self->bucket[0] == '\0') {
2144 device_set_error(pself,
2145 vstrallocf(_("Empty bucket name in device %s"), device_name),
2146 DEVICE_STATUS_DEVICE_ERROR);
2147 amfree(self->bucket);
2148 amfree(self->prefix);
2152 if (self->reps == NULL) {
2153 self->reps = g_strdup(S3_DEVICE_REPS_DEFAULT);
2156 if (self->reps_bucket == NULL) {
2157 self->reps_bucket = g_strdup(S3_DEVICE_REPS_BUCKET_DEFAULT);
2160 g_debug(_("S3 driver using bucket '%s', prefix '%s'"), self->bucket, self->prefix);
2162 /* default values */
2163 self->verbose = FALSE;
2164 self->s3_api = S3_API_S3;
2166 /* use SSL if available */
2167 self->use_ssl = s3_curl_supports_ssl();
2168 bzero(&tmp_value, sizeof(GValue));
2169 g_value_init(&tmp_value, G_TYPE_BOOLEAN);
2170 g_value_set_boolean(&tmp_value, self->use_ssl);
2171 device_set_simple_property(pself, device_property_s3_ssl.ID,
2172 &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
2174 /* reuse connection */
2175 self->reuse_connection = TRUE;
2176 bzero(&tmp_value, sizeof(GValue));
2177 g_value_init(&tmp_value, G_TYPE_BOOLEAN);
2178 g_value_set_boolean(&tmp_value, self->reuse_connection);
2179 device_set_simple_property(pself, device_property_reuse_connection.ID,
2180 &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
2182 /* Set default create_bucket */
2183 self->create_bucket = TRUE;
2184 bzero(&tmp_value, sizeof(GValue));
2185 g_value_init(&tmp_value, G_TYPE_BOOLEAN);
2186 g_value_set_boolean(&tmp_value, self->create_bucket);
2187 device_set_simple_property(pself, device_property_create_bucket.ID,
2188 &tmp_value, PROPERTY_SURETY_GOOD, PROPERTY_SOURCE_DEFAULT);
2190 if (parent_class->open_device) {
2191 parent_class->open_device(pself, device_name, device_type, device_node);
2195 static void s3_device_finalize(GObject * obj_self) {
2196 S3Device *self = S3_DEVICE (obj_self);
2199 if(G_OBJECT_CLASS(parent_class)->finalize)
2200 (* G_OBJECT_CLASS(parent_class)->finalize)(obj_self);
2202 if (self->thread_pool_delete) {
2203 g_thread_pool_free(self->thread_pool_delete, 1, 1);
2204 self->thread_pool_delete = NULL;
2206 if (self->thread_pool_write) {
2207 g_thread_pool_free(self->thread_pool_write, 1, 1);
2208 self->thread_pool_write = NULL;
2210 if (self->thread_pool_read) {
2211 g_thread_pool_free(self->thread_pool_read, 1, 1);
2212 self->thread_pool_read = NULL;
2214 if (self->thread_idle_mutex) {
2215 g_mutex_free(self->thread_idle_mutex);
2216 self->thread_idle_mutex = NULL;
2218 if (self->thread_idle_cond) {
2219 g_cond_free(self->thread_idle_cond);
2220 self->thread_idle_cond = NULL;
2223 for (thread = 0; thread < self->nb_threads; thread++) {
2224 g_mutex_free(self->s3t[thread].now_mutex);
2225 if(self->s3t[thread].s3) s3_free(self->s3t[thread].s3);
2226 g_free(self->s3t[thread].curl_buffer.buffer);
2230 if(self->bucket) g_free(self->bucket);
2231 if(self->prefix) g_free(self->prefix);
2232 if(self->access_key) g_free(self->access_key);
2233 if(self->secret_key) g_free(self->secret_key);
2234 if(self->swift_account_id) g_free(self->swift_account_id);
2235 if(self->swift_access_key) g_free(self->swift_access_key);
2236 if(self->username) g_free(self->username);
2237 if(self->password) g_free(self->password);
2238 if(self->tenant_id) g_free(self->tenant_id);
2239 if(self->tenant_name) g_free(self->tenant_name);
2240 if(self->host) g_free(self->host);
2241 if(self->service_path) g_free(self->service_path);
2242 if(self->user_token) g_free(self->user_token);
2243 if(self->bucket_location) g_free(self->bucket_location);
2244 if(self->storage_class) g_free(self->storage_class);
2245 if(self->server_side_encryption) g_free(self->server_side_encryption);
2246 if(self->proxy) g_free(self->proxy);
2247 if(self->ca_info) g_free(self->ca_info);
2248 if(self->reps) g_free(self->reps);
2249 if(self->reps_bucket) g_free(self->reps_bucket);
2252 static gboolean setup_handle(S3Device * self) {
2253 Device *d_self = DEVICE(self);
2255 guint response_code;
2256 s3_error_code_t s3_error_code;
2259 if (self->s3t == NULL) {
2260 if (self->s3_api == S3_API_S3) {
2261 if (self->access_key == NULL || self->access_key[0] == '\0') {
2262 device_set_error(d_self,
2263 g_strdup(_("No Amazon access key specified")),
2264 DEVICE_STATUS_DEVICE_ERROR);
2268 if (self->secret_key == NULL || self->secret_key[0] == '\0') {
2269 device_set_error(d_self,
2270 g_strdup(_("No Amazon secret key specified")),
2271 DEVICE_STATUS_DEVICE_ERROR);
2274 } else if (self->s3_api == S3_API_SWIFT_1) {
2275 if (self->swift_account_id == NULL ||
2276 self->swift_account_id[0] == '\0') {
2277 device_set_error(d_self,
2278 g_strdup(_("No Swift account id specified")),
2279 DEVICE_STATUS_DEVICE_ERROR);
2282 if (self->swift_access_key == NULL ||
2283 self->swift_access_key[0] == '\0') {
2284 device_set_error(d_self,
2285 g_strdup(_("No Swift access key specified")),
2286 DEVICE_STATUS_DEVICE_ERROR);
2289 } else if (self->s3_api == S3_API_SWIFT_2) {
2290 if (!((self->username && self->password && self->tenant_id) ||
2291 (self->username && self->password && self->tenant_name) ||
2292 (self->access_key && self->secret_key && self->tenant_id) ||
2293 (self->access_key && self->secret_key && self->tenant_name))) {
2294 device_set_error(d_self,
2295 g_strdup(_("Missing authorization properties")),
2296 DEVICE_STATUS_DEVICE_ERROR);
2299 } else if (self->s3_api == S3_API_OAUTH2) {
2300 if (self->client_id == NULL ||
2301 self->client_id[0] == '\0') {
2302 device_set_error(d_self,
2303 g_strdup(_("Missing client_id properties")),
2304 DEVICE_STATUS_DEVICE_ERROR);
2307 if (self->client_secret == NULL ||
2308 self->client_secret[0] == '\0') {
2309 device_set_error(d_self,
2310 g_strdup(_("Missing client_secret properties")),
2311 DEVICE_STATUS_DEVICE_ERROR);
2314 if (self->refresh_token == NULL ||
2315 self->refresh_token[0] == '\0') {
2316 device_set_error(d_self,
2317 g_strdup(_("Missing refresh_token properties")),
2318 DEVICE_STATUS_DEVICE_ERROR);
2321 if (self->project_id == NULL ||
2322 self->project_id[0] == '\0') {
2323 device_set_error(d_self,
2324 g_strdup(_("Missing project_id properties")),
2325 DEVICE_STATUS_DEVICE_ERROR);
2328 } else if (self->s3_api == S3_API_CASTOR) {
2329 self->use_s3_multi_delete = 0;
2330 self->use_subdomain = FALSE;
2331 if(self->service_path) {
2332 g_free(self->service_path);
2333 self->service_path = NULL;
2337 self->s3t = g_new0(S3_by_thread, self->nb_threads);
2338 if (self->s3t == NULL) {
2339 device_set_error(d_self,
2340 g_strdup(_("Can't allocate S3Handle array")),
2341 DEVICE_STATUS_DEVICE_ERROR);
2345 self->thread_idle_cond = g_cond_new();
2346 self->thread_idle_mutex = g_mutex_new();
2348 for (thread = 0; thread < self->nb_threads; thread++) {
2349 self->s3t[thread].idle = 1;
2350 self->s3t[thread].done = 1;
2351 self->s3t[thread].eof = FALSE;
2352 self->s3t[thread].errflags = DEVICE_STATUS_SUCCESS;
2353 self->s3t[thread].errmsg = NULL;
2354 self->s3t[thread].filename = NULL;
2355 self->s3t[thread].curl_buffer.buffer = NULL;
2356 self->s3t[thread].curl_buffer.buffer_len = 0;
2357 self->s3t[thread].now_mutex = g_mutex_new();
2358 self->s3t[thread].s3 = s3_open(self->access_key, self->secret_key,
2359 self->swift_account_id,
2360 self->swift_access_key,
2361 self->host, self->service_path,
2362 self->use_subdomain,
2363 self->user_token, self->bucket_location,
2364 self->storage_class, self->ca_info,
2365 self->server_side_encryption,
2373 self->client_secret,
2374 self->refresh_token,
2375 self->reuse_connection,
2376 self->reps, self->reps_bucket);
2377 if (self->s3t[thread].s3 == NULL) {
2378 device_set_error(d_self,
2379 stralloc(_("Internal error creating S3 handle")),
2380 DEVICE_STATUS_DEVICE_ERROR);
2381 self->nb_threads = thread+1;
2386 g_debug("Create %d threads", self->nb_threads);
2387 self->thread_pool_delete = g_thread_pool_new(s3_thread_delete_block,
2388 self, self->nb_threads, 0,
2390 self->thread_pool_write = g_thread_pool_new(s3_thread_write_block, self,
2391 self->nb_threads, 0, NULL);
2392 self->thread_pool_read = g_thread_pool_new(s3_thread_read_block, self,
2393 self->nb_threads, 0, NULL);
2395 for (thread = 0; thread < self->nb_threads; thread++) {
2396 s3_verbose(self->s3t[thread].s3, self->verbose);
2398 if (!s3_use_ssl(self->s3t[thread].s3, self->use_ssl)) {
2399 device_set_error(d_self, g_strdup_printf(_(
2400 "Error setting S3 SSL/TLS use "
2401 "(tried to enable SSL/TLS for S3, but curl doesn't support it?)")),
2402 DEVICE_STATUS_DEVICE_ERROR);
2406 if (self->max_send_speed &&
2407 !s3_set_max_send_speed(self->s3t[thread].s3,
2408 self->max_send_speed)) {
2409 device_set_error(d_self,
2410 g_strdup("Could not set S3 maximum send speed"),
2411 DEVICE_STATUS_DEVICE_ERROR);
2415 if (self->max_recv_speed &&
2416 !s3_set_max_recv_speed(self->s3t[thread].s3,
2417 self->max_recv_speed)) {
2418 device_set_error(d_self,
2419 g_strdup("Could not set S3 maximum recv speed"),
2420 DEVICE_STATUS_DEVICE_ERROR);
2425 for (thread = 0; thread < self->nb_threads; thread++) {
2426 if (!s3_open2(self->s3t[thread].s3)) {
2427 if (self->s3_api == S3_API_SWIFT_1 ||
2428 self->s3_api == S3_API_SWIFT_2) {
2429 s3_error(self->s3t[0].s3, NULL, &response_code,
2430 &s3_error_code, NULL, &curl_code, NULL);
2431 device_set_error(d_self,
2432 g_strdup_printf(_("s3_open2 failed: %s"),
2433 s3_strerror(self->s3t[0].s3)),
2434 DEVICE_STATUS_DEVICE_ERROR);
2435 self->nb_threads = thread+1;
2438 device_set_error(d_self,
2439 g_strdup("s3_open2 failed"),
2440 DEVICE_STATUS_DEVICE_ERROR);
2454 S3Device *self = S3_DEVICE(pself);
2455 guint response_code;
2456 s3_error_code_t s3_error_code;
2459 if (s3_is_bucket_exists(self->s3t[0].s3, self->bucket, self->project_id)) {
2463 s3_error(self->s3t[0].s3, NULL, &response_code, &s3_error_code, NULL, &curl_code, NULL);
2465 if (response_code == 0 && s3_error_code == 0 &&
2466 (curl_code == CURLE_COULDNT_CONNECT ||
2467 curl_code == CURLE_COULDNT_RESOLVE_HOST)) {
2468 device_set_error(pself,
2469 g_strdup_printf(_("While connecting to S3 bucket: %s"),
2470 s3_strerror(self->s3t[0].s3)),
2471 DEVICE_STATUS_DEVICE_ERROR);
2475 if (!self->create_bucket) {
2476 device_set_error(pself,
2477 g_strdup_printf(_("Can't list bucket: %s"),
2478 s3_strerror(self->s3t[0].s3)),
2479 DEVICE_STATUS_DEVICE_ERROR);
2483 if (!s3_make_bucket(self->s3t[0].s3, self->bucket, self->project_id)) {
2484 s3_error(self->s3t[0].s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
2486 /* if it isn't an expected error (bucket already exists),
2488 if (response_code != 409 ||
2489 (s3_error_code != S3_ERROR_BucketAlreadyExists &&
2490 s3_error_code != S3_ERROR_BucketAlreadyOwnedByYou)) {
2491 device_set_error(pself,
2492 g_strdup_printf(_("While creating new S3 bucket: %s"), s3_strerror(self->s3t[0].s3)),
2493 DEVICE_STATUS_DEVICE_ERROR);
2500 static int progress_func(
2502 double dltotal G_GNUC_UNUSED,
2504 double ultotal G_GNUC_UNUSED,
2507 S3_by_thread *s3t = (S3_by_thread *)thread_data;
2509 g_mutex_lock(s3t->now_mutex);
2512 g_mutex_unlock(s3t->now_mutex);
2517 static DeviceStatusFlags
2518 s3_device_read_label(Device *pself) {
2519 S3Device *self = S3_DEVICE(pself);
2521 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
2522 dumpfile_t *amanda_header;
2523 /* note that this may be called from s3_device_start, when
2524 * self->access_mode is not ACCESS_NULL */
2526 amfree(pself->volume_label);
2527 amfree(pself->volume_time);
2528 dumpfile_free(pself->volume_header);
2529 pself->volume_header = NULL;
2531 if (device_in_error(self)) return pself->status;
2533 if (!setup_handle(self)) {
2534 /* setup_handle already set our error message */
2535 return pself->status;
2539 key = special_file_to_key(self, "tapestart", -1);
2541 if (!make_bucket(pself)) {
2542 return pself->status;
2545 if (!s3_read(self->s3t[0].s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL)) {
2546 guint response_code;
2547 s3_error_code_t s3_error_code;
2548 s3_error(self->s3t[0].s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
2550 /* if it's an expected error (not found), just return FALSE */
2551 if (response_code == 404 &&
2552 (s3_error_code == S3_ERROR_None ||
2553 s3_error_code == S3_ERROR_Unknown ||
2554 s3_error_code == S3_ERROR_NoSuchKey ||
2555 s3_error_code == S3_ERROR_NoSuchEntity ||
2556 s3_error_code == S3_ERROR_NoSuchBucket)) {
2557 g_debug(_("Amanda header not found while reading tapestart header (this is expected for empty tapes)"));
2558 device_set_error(pself,
2559 stralloc(_("Amanda header not found -- unlabeled volume?")),
2560 DEVICE_STATUS_DEVICE_ERROR
2561 | DEVICE_STATUS_VOLUME_ERROR
2562 | DEVICE_STATUS_VOLUME_UNLABELED);
2563 return pself->status;
2566 /* otherwise, log it and return */
2567 device_set_error(pself,
2568 vstrallocf(_("While trying to read tapestart header: %s"), s3_strerror(self->s3t[0].s3)),
2569 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
2570 return pself->status;
2573 /* handle an empty file gracefully */
2574 if (buf.buffer_len == 0) {
2575 device_set_error(pself, stralloc(_("Empty header file")), DEVICE_STATUS_VOLUME_ERROR);
2576 return pself->status;
2579 pself->header_block_size = buf.buffer_len;
2580 g_assert(buf.buffer != NULL);
2581 amanda_header = g_new(dumpfile_t, 1);
2582 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
2583 pself->volume_header = amanda_header;
2586 if (amanda_header->type != F_TAPESTART) {
2587 device_set_error(pself, stralloc(_("Invalid amanda header")), DEVICE_STATUS_VOLUME_ERROR);
2588 return pself->status;
2591 pself->volume_label = g_strdup(amanda_header->name);
2592 pself->volume_time = g_strdup(amanda_header->datestamp);
2593 /* pself->volume_header is already set */
2595 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
2597 return pself->status;
2601 s3_device_start (Device * pself, DeviceAccessMode mode,
2602 char * label, char * timestamp) {
2605 guint64 total_size = 0;
2608 self = S3_DEVICE(pself);
2610 if (device_in_error(self)) return FALSE;
2612 if (!setup_handle(self)) {
2613 /* setup_handle already set our error message */
2618 pself->access_mode = mode;
2619 g_mutex_lock(pself->device_mutex);
2620 pself->in_file = FALSE;
2621 g_mutex_unlock(pself->device_mutex);
2623 /* try creating the bucket, in case it doesn't exist */
2624 if (!make_bucket(pself)) {
2628 /* take care of any dirty work for this mode */
2631 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
2632 /* s3_device_read_label already set our error message */
2638 if (!delete_all_files(self)) {
2642 /* write a new amanda header */
2643 if (!write_amanda_header(self, label, timestamp)) {
2647 pself->volume_label = newstralloc(pself->volume_label, label);
2648 pself->volume_time = newstralloc(pself->volume_time, timestamp);
2650 /* unset the VOLUME_UNLABELED flag, if it was set */
2651 device_set_error(pself, NULL, DEVICE_STATUS_SUCCESS);
2655 if (pself->volume_label == NULL && s3_device_read_label(pself) != DEVICE_STATUS_SUCCESS) {
2656 /* s3_device_read_label already set our error message */
2659 result = s3_list_keys(self->s3t[0].s3, self->bucket, NULL, NULL, &keys, &total_size);
2661 device_set_error(pself,
2662 vstrallocf(_("While listing S3 keys: %s"), s3_strerror(self->s3t[0].s3)),
2663 DEVICE_STATUS_DEVICE_ERROR|DEVICE_STATUS_VOLUME_ERROR);
2666 self->volume_bytes = total_size;
2669 return seek_to_end(self);
2673 g_assert_not_reached();
2683 S3Device *self = S3_DEVICE(pself);
2687 /* we're not in a file anymore */
2688 pself->access_mode = ACCESS_NULL;
2690 if (device_in_error(pself)) return FALSE;
2695 /* functions for writing */
2699 s3_device_get_bytes_read(
2702 S3Device *self = S3_DEVICE(pself);
2706 g_mutex_unlock(pself->device_mutex);
2707 /* Add per thread */
2708 g_mutex_lock(self->thread_idle_mutex);
2709 dltotal = self->dltotal;
2710 for (thread = 0; thread < self->nb_threads_recovery; thread++) {
2711 g_mutex_lock(self->s3t[thread].now_mutex);
2712 dltotal += self->s3t[thread].dlnow;
2713 g_mutex_unlock(self->s3t[thread].now_mutex);
2715 g_mutex_unlock(self->thread_idle_mutex);
2716 g_mutex_lock(pself->device_mutex);
2723 s3_device_get_bytes_written(
2726 S3Device *self = S3_DEVICE(pself);
2730 g_mutex_unlock(pself->device_mutex);
2731 /* Add per thread */
2732 g_mutex_lock(self->thread_idle_mutex);
2733 ultotal = self->ultotal;
2734 for (thread = 0; thread < self->nb_threads_backup; thread++) {
2735 g_mutex_lock(self->s3t[thread].now_mutex);
2736 ultotal += self->s3t[thread].ulnow;
2737 g_mutex_unlock(self->s3t[thread].now_mutex);
2739 g_mutex_unlock(self->thread_idle_mutex);
2740 g_mutex_lock(pself->device_mutex);
2747 s3_device_start_file (Device *pself, dumpfile_t *jobInfo) {
2748 S3Device *self = S3_DEVICE(pself);
2749 CurlBuffer amanda_header = {NULL, 0, 0, 0};
2755 if (device_in_error(self)) return FALSE;
2758 pself->is_eom = FALSE;
2760 /* Set the blocksize to zero, since there's no header to skip (it's stored
2761 * in a distinct file, rather than block zero) */
2762 jobInfo->blocksize = 0;
2764 /* Build the amanda header. */
2765 header_size = 0; /* no minimum size */
2766 amanda_header.buffer = device_build_amanda_header(pself, jobInfo,
2768 if (amanda_header.buffer == NULL) {
2769 device_set_error(pself,
2770 stralloc(_("Amanda file header won't fit in a single block!")),
2771 DEVICE_STATUS_DEVICE_ERROR);
2774 amanda_header.buffer_len = header_size;
2776 if(check_at_leom(self, header_size))
2777 pself->is_eom = TRUE;
2779 if(check_at_peom(self, header_size)) {
2780 pself->is_eom = TRUE;
2781 device_set_error(pself,
2782 stralloc(_("No space left on device")),
2783 DEVICE_STATUS_DEVICE_ERROR);
2784 g_free(amanda_header.buffer);
2788 for (thread = 0; thread < self->nb_threads; thread++) {
2789 self->s3t[thread].idle = 1;
2790 self->s3t[thread].ulnow = 0;
2793 /* set the file and block numbers correctly */
2794 pself->file = (pself->file > 0)? pself->file+1 : 1;
2796 g_mutex_lock(pself->device_mutex);
2797 pself->in_file = TRUE;
2798 pself->bytes_written = 0;
2799 g_mutex_unlock(pself->device_mutex);
2800 g_mutex_lock(self->thread_idle_mutex);
2802 g_mutex_unlock(self->thread_idle_mutex);
2803 /* write it out as a special block (not the 0th) */
2804 key = special_file_to_key(self, "filestart", pself->file);
2805 result = s3_upload(self->s3t[0].s3, self->bucket, key, S3_BUFFER_READ_FUNCS,
2806 &amanda_header, NULL, NULL);
2807 g_free(amanda_header.buffer);
2810 device_set_error(pself,
2811 vstrallocf(_("While writing filestart header: %s"), s3_strerror(self->s3t[0].s3)),
2812 DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR);
2816 self->volume_bytes += header_size;
2822 s3_device_write_block (Device * pself, guint size, gpointer data) {
2824 S3Device * self = S3_DEVICE(pself);
2825 int idle_thread = 0;
2827 int first_idle = -1;
2829 g_assert (self != NULL);
2830 g_assert (data != NULL);
2831 if (device_in_error(self)) return FALSE;
2833 if(check_at_leom(self, size))
2834 pself->is_eom = TRUE;
2836 if(check_at_peom(self, size)) {
2837 pself->is_eom = TRUE;
2838 device_set_error(pself,
2839 stralloc(_("No space left on device")),
2840 DEVICE_STATUS_DEVICE_ERROR);
2844 filename = file_and_block_to_key(self, pself->file, pself->block);
2846 g_mutex_lock(self->thread_idle_mutex);
2847 while (!idle_thread) {
2849 for (thread = 0; thread < self->nb_threads_backup; thread++) {
2850 if (self->s3t[thread].idle == 1) {
2852 /* Check if the thread is in error */
2853 if (self->s3t[thread].errflags != DEVICE_STATUS_SUCCESS) {
2854 device_set_error(pself, (char *)self->s3t[thread].errmsg,
2855 self->s3t[thread].errflags);
2856 self->s3t[thread].errflags = DEVICE_STATUS_SUCCESS;
2857 self->s3t[thread].errmsg = NULL;
2858 g_mutex_unlock(self->thread_idle_mutex);
2861 if (first_idle == -1) {
2862 first_idle = thread;
2868 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
2871 thread = first_idle;
2873 self->s3t[thread].idle = 0;
2874 self->s3t[thread].done = 0;
2875 if (self->s3t[thread].curl_buffer.buffer &&
2876 self->s3t[thread].curl_buffer.buffer_len < size) {
2877 g_free((char *)self->s3t[thread].curl_buffer.buffer);
2878 self->s3t[thread].curl_buffer.buffer = NULL;
2879 self->s3t[thread].curl_buffer.buffer_len = 0;
2880 self->s3t[thread].buffer_len = 0;
2882 if (self->s3t[thread].curl_buffer.buffer == NULL) {
2883 self->s3t[thread].curl_buffer.buffer = g_malloc(size);
2884 self->s3t[thread].curl_buffer.buffer_len = size;
2885 self->s3t[thread].buffer_len = size;
2887 memcpy((char *)self->s3t[thread].curl_buffer.buffer, data, size);
2888 self->s3t[thread].curl_buffer.buffer_pos = 0;
2889 self->s3t[thread].curl_buffer.buffer_len = size;
2890 self->s3t[thread].curl_buffer.max_buffer_size = 0;
2891 self->s3t[thread].filename = filename;
2892 g_thread_pool_push(self->thread_pool_write, &self->s3t[thread], NULL);
2893 g_mutex_unlock(self->thread_idle_mutex);
2896 self->volume_bytes += size;
2901 s3_thread_write_block(
2902 gpointer thread_data,
2905 S3_by_thread *s3t = (S3_by_thread *)thread_data;
2906 Device *pself = (Device *)data;
2907 S3Device *self = S3_DEVICE(pself);
2910 result = s3_upload(s3t->s3, self->bucket, (char *)s3t->filename,
2911 S3_BUFFER_READ_FUNCS, (CurlBuffer *)&s3t->curl_buffer,
2912 progress_func, s3t);
2913 g_free((void *)s3t->filename);
2914 s3t->filename = NULL;
2916 s3t->errflags = DEVICE_STATUS_DEVICE_ERROR | DEVICE_STATUS_VOLUME_ERROR;
2917 s3t->errmsg = g_strdup_printf(_("While writing data block to S3: %s"), s3_strerror(s3t->s3));
2919 g_mutex_lock(self->thread_idle_mutex);
2923 self->ultotal += s3t->curl_buffer.buffer_len;
2924 s3t->curl_buffer.buffer_len = s3t->buffer_len;
2926 g_cond_broadcast(self->thread_idle_cond);
2927 g_mutex_unlock(self->thread_idle_mutex);
2931 s3_device_finish_file (Device * pself) {
2932 S3Device *self = S3_DEVICE(pself);
2934 /* Check all threads are done */
2935 int idle_thread = 0;
2938 g_mutex_lock(self->thread_idle_mutex);
2939 while (idle_thread != self->nb_threads) {
2941 for (thread = 0; thread < self->nb_threads; thread++) {
2942 if (self->s3t[thread].idle == 1) {
2945 /* check thread status */
2946 if (self->s3t[thread].errflags != DEVICE_STATUS_SUCCESS) {
2947 device_set_error(pself, (char *)self->s3t[thread].errmsg,
2948 self->s3t[thread].errflags);
2949 self->s3t[thread].errflags = DEVICE_STATUS_SUCCESS;
2950 self->s3t[thread].errmsg = NULL;
2953 if (idle_thread != self->nb_threads) {
2954 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
2958 g_mutex_unlock(self->thread_idle_mutex);
2960 if (device_in_error(pself)) return FALSE;
2962 /* we're not in a file anymore */
2963 g_mutex_lock(pself->device_mutex);
2964 pself->in_file = FALSE;
2965 pself->bytes_written = 0;;
2966 g_mutex_unlock(pself->device_mutex);
2972 s3_device_recycle_file(Device *pself, guint file) {
2973 S3Device *self = S3_DEVICE(pself);
2974 if (device_in_error(self)) return FALSE;
2977 delete_file(self, file);
2978 s3_wait_thread_delete(self);
2979 return !device_in_error(self);
2980 /* delete_file already set our error message if necessary */
2984 s3_device_erase(Device *pself) {
2985 S3Device *self = S3_DEVICE(pself);
2987 const char *errmsg = NULL;
2988 guint response_code;
2989 s3_error_code_t s3_error_code;
2991 if (!setup_handle(self)) {
2992 /* error set by setup_handle */
2997 key = special_file_to_key(self, "tapestart", -1);
2998 if (!s3_delete(self->s3t[0].s3, self->bucket, key)) {
2999 s3_error(self->s3t[0].s3, &errmsg, NULL, NULL, NULL, NULL, NULL);
3000 device_set_error(pself,
3002 DEVICE_STATUS_DEVICE_ERROR);
3007 dumpfile_free(pself->volume_header);
3008 pself->volume_header = NULL;
3010 if (!delete_all_files(self))
3013 device_set_error(pself, g_strdup("Unlabeled volume"),
3014 DEVICE_STATUS_VOLUME_UNLABELED);
3016 if (self->create_bucket &&
3017 !s3_delete_bucket(self->s3t[0].s3, self->bucket)) {
3018 s3_error(self->s3t[0].s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
3021 * ignore the error if the bucket isn't empty (there may be data from elsewhere)
3022 * or the bucket not existing (already deleted perhaps?)
3025 (response_code == 409 && s3_error_code == S3_ERROR_BucketNotEmpty) ||
3026 (response_code == 404 && s3_error_code == S3_ERROR_NoSuchBucket))) {
3028 device_set_error(pself,
3030 DEVICE_STATUS_DEVICE_ERROR);
3034 self->volume_bytes = 0;
3038 /* functions for reading */
3041 s3_device_seek_file(Device *pself, guint file) {
3042 S3Device *self = S3_DEVICE(pself);
3045 CurlBuffer buf = {NULL, 0, 0, S3_DEVICE_MAX_BLOCK_SIZE};
3046 dumpfile_t *amanda_header;
3047 const char *errmsg = NULL;
3050 if (device_in_error(self)) return NULL;
3055 pself->is_eof = FALSE;
3057 g_mutex_lock(pself->device_mutex);
3058 pself->in_file = FALSE;
3059 pself->bytes_read = 0;
3060 g_mutex_unlock(pself->device_mutex);
3061 self->next_block_to_read = 0;
3062 g_mutex_lock(self->thread_idle_mutex);
3064 g_mutex_unlock(self->thread_idle_mutex);
3067 key = special_file_to_key(self, "filestart", pself->file);
3068 result = s3_read(self->s3t[0].s3, self->bucket, key, S3_BUFFER_WRITE_FUNCS,
3073 guint response_code;
3074 s3_error_code_t s3_error_code;
3075 s3_error(self->s3t[0].s3, &errmsg, &response_code, &s3_error_code, NULL, NULL, NULL);
3077 /* if it's an expected error (not found), check what to do. */
3078 if (response_code == 404 &&
3079 (s3_error_code == S3_ERROR_None ||
3080 s3_error_code == S3_ERROR_NoSuchKey ||
3081 s3_error_code == S3_ERROR_NoSuchEntity)) {
3083 next_file = find_next_file(self, pself->file);
3084 if (next_file > 0) {
3085 /* Note short-circut of dispatcher. */
3086 return s3_device_seek_file(pself, next_file);
3087 } else if (next_file == 0) {
3088 /* No next file. Check if we are one past the end. */
3089 key = special_file_to_key(self, "filestart", pself->file - 1);
3090 result = s3_read(self->s3t[0].s3, self->bucket, key,
3091 S3_BUFFER_WRITE_FUNCS, &buf, NULL, NULL);
3094 /* pself->file, etc. are already correct */
3095 return make_tapeend_header();
3097 device_set_error(pself,
3098 stralloc(_("Attempt to read past tape-end file")),
3099 DEVICE_STATUS_SUCCESS);
3104 /* An unexpected error occured finding out if we are the last file. */
3105 device_set_error(pself,
3107 DEVICE_STATUS_DEVICE_ERROR);
3112 /* and make a dumpfile_t out of it */
3113 g_assert(buf.buffer != NULL);
3114 amanda_header = g_new(dumpfile_t, 1);
3115 fh_init(amanda_header);
3116 parse_file_header(buf.buffer, amanda_header, buf.buffer_pos);
3119 switch (amanda_header->type) {
3121 case F_CONT_DUMPFILE:
3122 case F_SPLIT_DUMPFILE:
3126 device_set_error(pself,
3127 stralloc(_("Invalid amanda header while reading file header")),
3128 DEVICE_STATUS_VOLUME_ERROR);
3129 g_free(amanda_header);
3133 for (thread = 0; thread < self->nb_threads; thread++) {
3134 self->s3t[thread].idle = 1;
3135 self->s3t[thread].eof = FALSE;
3136 self->s3t[thread].ulnow = 0;
3138 g_mutex_lock(pself->device_mutex);
3139 pself->in_file = TRUE;
3140 g_mutex_unlock(pself->device_mutex);
3141 return amanda_header;
3145 s3_device_seek_block(Device *pself, guint64 block) {
3146 S3Device * self = S3_DEVICE(pself);
3147 if (device_in_error(pself)) return FALSE;
3150 pself->block = block;
3151 self->next_block_to_read = block;
3156 s3_device_read_block (Device * pself, gpointer data, int *size_req) {
3157 S3Device * self = S3_DEVICE(pself);
3162 g_assert (self != NULL);
3163 if (device_in_error(self)) return -1;
3165 g_mutex_lock(self->thread_idle_mutex);
3166 /* start a read ahead for each thread */
3167 for (thread = 0; thread < self->nb_threads_recovery; thread++) {
3168 S3_by_thread *s3t = &self->s3t[thread];
3170 key = file_and_block_to_key(self, pself->file, self->next_block_to_read);
3171 s3t->filename = key;
3177 s3t->errflags = DEVICE_STATUS_SUCCESS;
3178 if (self->s3t[thread].curl_buffer.buffer &&
3179 (int)self->s3t[thread].curl_buffer.buffer_len < *size_req) {
3180 g_free(self->s3t[thread].curl_buffer.buffer);
3181 self->s3t[thread].curl_buffer.buffer = NULL;
3182 self->s3t[thread].curl_buffer.buffer_len = 0;
3183 self->s3t[thread].buffer_len = 0;
3185 if (!self->s3t[thread].curl_buffer.buffer) {
3186 self->s3t[thread].curl_buffer.buffer = g_malloc(*size_req);
3187 self->s3t[thread].curl_buffer.buffer_len = *size_req;
3188 self->s3t[thread].buffer_len = *size_req;
3190 s3t->curl_buffer.buffer_pos = 0;
3191 s3t->curl_buffer.max_buffer_size = S3_DEVICE_MAX_BLOCK_SIZE;
3192 self->next_block_to_read++;
3193 g_thread_pool_push(self->thread_pool_read, s3t, NULL);
3198 key = file_and_block_to_key(self, pself->file, pself->block);
3199 g_assert(key != NULL);
3201 /* find which thread read the key */
3202 for (thread = 0; thread < self->nb_threads_recovery; thread++) {
3204 s3t = &self->s3t[thread];
3207 strcmp(key, (char *)s3t->filename) == 0) {
3211 pself->is_eof = TRUE;
3212 g_mutex_lock(pself->device_mutex);
3213 pself->in_file = FALSE;
3214 g_mutex_unlock(pself->device_mutex);
3215 device_set_error(pself, stralloc(_("EOF")),
3216 DEVICE_STATUS_SUCCESS);
3217 g_mutex_unlock(self->thread_idle_mutex);
3219 } else if (s3t->errflags != DEVICE_STATUS_SUCCESS) {
3220 /* return the error */
3221 device_set_error(pself, (char *)s3t->errmsg, s3t->errflags);
3223 g_mutex_unlock(self->thread_idle_mutex);
3226 } else if ((guint)*size_req >= s3t->curl_buffer.buffer_pos) {
3227 /* return the buffer */
3228 g_mutex_unlock(self->thread_idle_mutex);
3229 memcpy(data, s3t->curl_buffer.buffer,
3230 s3t->curl_buffer.buffer_pos);
3231 *size_req = s3t->curl_buffer.buffer_pos;
3234 g_free((char *)s3t->filename);
3237 g_mutex_lock(self->thread_idle_mutex);
3239 } else { /* buffer not enough large */
3240 *size_req = s3t->curl_buffer.buffer_len;
3242 g_mutex_unlock(self->thread_idle_mutex);
3248 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
3252 /* start a read ahead for the thread */
3253 for (thread = 0; thread < self->nb_threads_recovery; thread++) {
3254 S3_by_thread *s3t = &self->s3t[thread];
3256 key = file_and_block_to_key(self, pself->file, self->next_block_to_read);
3257 s3t->filename = key;
3263 s3t->errflags = DEVICE_STATUS_SUCCESS;
3264 if (!self->s3t[thread].curl_buffer.buffer) {
3265 self->s3t[thread].curl_buffer.buffer = g_malloc(*size_req);
3266 self->s3t[thread].curl_buffer.buffer_len = *size_req;
3268 s3t->curl_buffer.buffer_pos = 0;
3269 self->next_block_to_read++;
3270 g_thread_pool_push(self->thread_pool_read, s3t, NULL);
3273 g_mutex_unlock(self->thread_idle_mutex);
3280 s3_thread_read_block(
3281 gpointer thread_data,
3284 S3_by_thread *s3t = (S3_by_thread *)thread_data;
3285 Device *pself = (Device *)data;
3286 S3Device *self = S3_DEVICE(pself);
3289 result = s3_read(s3t->s3, self->bucket, (char *)s3t->filename,
3290 S3_BUFFER_WRITE_FUNCS,
3291 (CurlBuffer *)&s3t->curl_buffer, progress_func, s3t);
3293 g_mutex_lock(self->thread_idle_mutex);
3295 guint response_code;
3296 s3_error_code_t s3_error_code;
3297 s3_error(s3t->s3, NULL, &response_code, &s3_error_code, NULL, NULL, NULL);
3298 /* if it's an expected error (not found), just return -1 */
3299 if (response_code == 404 &&
3300 (s3_error_code == S3_ERROR_None ||
3301 s3_error_code == S3_ERROR_Unknown ||
3302 s3_error_code == S3_ERROR_NoSuchKey ||
3303 s3_error_code == S3_ERROR_NoSuchEntity)) {
3307 /* otherwise, log it and return FALSE */
3308 s3t->errflags = DEVICE_STATUS_VOLUME_ERROR;
3309 s3t->errmsg = g_strdup_printf(_("While reading data block from S3: %s"),
3310 s3_strerror(s3t->s3));
3313 self->dltotal += s3t->curl_buffer.buffer_len;
3318 g_cond_broadcast(self->thread_idle_cond);
3319 g_mutex_unlock(self->thread_idle_mutex);
3325 check_at_peom(S3Device *self, guint64 size)
3327 if(self->enforce_volume_limit && (self->volume_limit > 0)) {
3328 guint64 newtotal = self->volume_bytes + size;
3329 if(newtotal > self->volume_limit) {
3337 check_at_leom(S3Device *self, guint64 size)
3339 guint64 block_size = DEVICE(self)->block_size;
3340 guint64 eom_warning_buffer = block_size *
3341 (EOM_EARLY_WARNING_ZONE_BLOCKS + self->nb_threads);
3346 if(self->enforce_volume_limit && (self->volume_limit > 0)) {
3347 guint64 newtotal = self->volume_bytes + size + eom_warning_buffer;
3348 if(newtotal > self->volume_limit) {
3362 if (self->thread_idle_mutex) {
3363 g_mutex_lock(self->thread_idle_mutex);
3364 while(nb_done != self->nb_threads) {
3366 for (thread = 0; thread < self->nb_threads; thread++) {
3367 if (self->s3t[thread].done == 1)
3370 if (nb_done != self->nb_threads) {
3371 g_cond_wait(self->thread_idle_cond, self->thread_idle_mutex);
3374 g_mutex_unlock(self->thread_idle_mutex);