2 * Copyright (c) 2005-2008 Zmanda Inc. All Rights Reserved.
4 * This library is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU Lesser General Public License version 2.1 as
6 * published by the Free Software Foundation.
8 * This library is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
13 * You should have received a copy of the GNU Lesser General Public License
14 * along with this library; if not, write to the Free Software Foundation,
15 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 * Contact information: Zmanda Inc., 465 S Mathlida Ave, Suite 300
18 * Sunnyvale, CA 94086, USA, or: http://www.zmanda.com
22 * - collect speed statistics
27 /* use a relative path here to avoid conflicting with Perl's config.h. */
28 #include "../config/config.h"
36 #ifdef HAVE_SYS_TYPES_H
37 #include <sys/types.h>
39 #ifdef HAVE_SYS_STAT_H
58 #include <curl/curl.h>
60 /* Constant renamed after version 7.10.7 */
61 #ifndef CURLINFO_RESPONSE_CODE
62 #define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
65 /* We don't need OpenSSL's kerberos support, and it's broken in
67 #define OPENSSL_NO_KRB5
69 #ifdef HAVE_OPENSSL_HMAC_H
70 # include <openssl/hmac.h>
72 # ifdef HAVE_CRYPTO_HMAC_H
73 # include <crypto/hmac.h>
81 #include <openssl/err.h>
82 #include <openssl/ssl.h>
83 #include <openssl/md5.h>
85 /* Maximum key length as specified in the S3 documentation
86 * (*excluding* null terminator) */
87 #define S3_MAX_KEY_LENGTH 1024
89 #define AMAZON_SECURITY_HEADER "x-amz-security-token"
90 #define AMAZON_BUCKET_CONF_TEMPLATE "\
91 <CreateBucketConfiguration>\n\
92 <LocationConstraint>%s</LocationConstraint>\n\
93 </CreateBucketConfiguration>"
95 /* parameters for exponential backoff in the face of retriable errors */
98 #define EXPONENTIAL_BACKOFF_START_USEC G_USEC_PER_SEC/100
99 /* double at each retry */
100 #define EXPONENTIAL_BACKOFF_BASE 2
101 /* retry 14 times (for a total of about 3 minutes spent waiting) */
102 #define EXPONENTIAL_BACKOFF_MAX_RETRIES 14
104 /* general "reasonable size" parameters */
105 #define MAX_ERROR_RESPONSE_LEN (100*1024)
107 /* Results which should always be retried */
108 #define RESULT_HANDLING_ALWAYS_RETRY \
109 { 400, S3_ERROR_RequestTimeout, 0, S3_RESULT_RETRY }, \
110 { 404, S3_ERROR_NoSuchBucket, 0, S3_RESULT_RETRY }, \
111 { 409, S3_ERROR_OperationAborted, 0, S3_RESULT_RETRY }, \
112 { 412, S3_ERROR_PreconditionFailed, 0, S3_RESULT_RETRY }, \
113 { 500, S3_ERROR_InternalError, 0, S3_RESULT_RETRY }, \
114 { 501, S3_ERROR_NotImplemented, 0, S3_RESULT_RETRY }, \
115 { 0, 0, CURLE_COULDNT_CONNECT, S3_RESULT_RETRY }, \
116 { 0, 0, CURLE_PARTIAL_FILE, S3_RESULT_RETRY }, \
117 { 0, 0, CURLE_OPERATION_TIMEOUTED, S3_RESULT_RETRY }, \
118 { 0, 0, CURLE_SEND_ERROR, S3_RESULT_RETRY }, \
119 { 0, 0, CURLE_RECV_ERROR, S3_RESULT_RETRY }, \
120 { 0, 0, CURLE_GOT_NOTHING, S3_RESULT_RETRY }
123 * Data structures and associated functions
127 /* (all strings in this struct are freed by s3_free()) */
133 char *bucket_location;
140 /* information from the last request */
142 guint last_response_code;
143 s3_error_code_t last_s3_error_code;
144 CURLcode last_curl_code;
145 guint last_num_retries;
146 void *last_response_body;
147 guint last_response_body_size;
152 s3_write_func write_func;
153 s3_reset_func reset_func;
156 gboolean headers_done;
160 /* Callback function to examine headers one-at-a-time
162 * @note this is the same as CURLOPT_HEADERFUNCTION
164 * @param data: The pointer to read data from
165 * @param size: The size of each "element" of the data buffer in bytes
166 * @param nmemb: The number of elements in the data buffer.
167 * So, the buffer's size is size*nmemb bytes.
168 * @param stream: the header_data (an opaque pointer)
170 * @return The number of bytes written to the buffer or
171 * CURL_WRITEFUNC_PAUSE to pause.
172 * If it's the number of bytes written, it should match the buffer size
174 typedef size_t (*s3_header_func)(void *data, size_t size, size_t nmemb, void *stream);
180 /* (see preprocessor magic in s3.h) */
182 static char * s3_error_code_names[] = {
183 #define S3_ERROR(NAME) #NAME
188 /* Convert an s3 error name to an error code. This function
189 * matches strings case-insensitively, and is appropriate for use
190 * on data from the network.
192 * @param s3_error_code: the error name
193 * @returns: the error code (see constants in s3.h)
195 static s3_error_code_t
196 s3_error_code_from_name(char *s3_error_name);
198 /* Convert an s3 error code to a string
200 * @param s3_error_code: the error code to convert
201 * @returns: statically allocated string
204 s3_error_name_from_code(s3_error_code_t s3_error_code);
210 /* result handling is specified by a static array of result_handling structs,
211 * which match based on response_code (from HTTP) and S3 error code. The result
212 * given for the first match is used. 0 acts as a wildcard for both response_code
213 * and s3_error_code. The list is terminated with a struct containing 0 for both
214 * response_code and s3_error_code; the result for that struct is the default
217 * See RESULT_HANDLING_ALWAYS_RETRY for an example.
220 S3_RESULT_RETRY = -1,
225 typedef struct result_handling {
227 s3_error_code_t s3_error_code;
232 /* Lookup a result in C{result_handling}.
234 * @param result_handling: array of handling specifications
235 * @param response_code: response code from operation
236 * @param s3_error_code: s3 error code from operation, if any
237 * @param curl_code: the CURL error, if any
238 * @returns: the matching result
241 lookup_result(const result_handling_t *result_handling,
243 s3_error_code_t s3_error_code,
247 * Precompiled regular expressions */
248 static regex_t etag_regex, error_name_regex, message_regex, subdomain_regex,
255 /* Construct the URL for an Amazon S3 REST request.
257 * A new string is allocated and returned; it is the responsiblity of the caller.
259 * @param hdl: the S3Handle object
260 * @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
261 * @param bucket: the bucket being accessed, or NULL for none
262 * @param key: the key being accessed, or NULL for none
263 * @param subresource: the sub-resource being accessed (e.g. "acl"), or NULL for none
264 * @param use_subdomain: if TRUE, a subdomain of s3.amazonaws.com will be used
267 build_url(const char *bucket,
269 const char *subresource,
271 gboolean use_subdomain,
274 /* Create proper authorization headers for an Amazon S3 REST
275 * request to C{headers}.
277 * @note: C{X-Amz} headers (in C{headers}) must
279 * - be in alphabetical order
280 * - have no spaces around the colon
281 * (don't yell at me -- see the Amazon Developer Guide)
283 * @param hdl: the S3Handle object
284 * @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
285 * @param bucket: the bucket being accessed, or NULL for none
286 * @param key: the key being accessed, or NULL for none
287 * @param subresource: the sub-resource being accessed (e.g. "acl"), or NULL for none
288 * @param md5_hash: the MD5 hash of the request body, or NULL for none
289 * @param use_subdomain: if TRUE, a subdomain of s3.amazonaws.com will be used
291 static struct curl_slist *
292 authenticate_request(S3Handle *hdl,
296 const char *subresource,
297 const char *md5_hash,
298 gboolean use_subdomain);
302 /* Interpret the response to an S3 operation, assuming CURL completed its request
303 * successfully. This function fills in the relevant C{hdl->last*} members.
305 * @param hdl: The S3Handle object
306 * @param body: the response body
307 * @param body_len: the length of the response body
308 * @param etag: The response's ETag header
309 * @param content_md5: The hex-encoded MD5 hash of the request body,
310 * which will be checked against the response's ETag header.
311 * If NULL, the header is not checked.
312 * If non-NULL, then the body should have the response headers at its beginnning.
313 * @returns: TRUE if the response should be retried (e.g., network error)
316 interpret_response(S3Handle *hdl,
318 char *curl_error_buffer,
322 const char *content_md5);
324 /* Perform an S3 operation. This function handles all of the details
325 * of retryig requests and so on.
327 * The concepts of bucket and keys are defined by the Amazon S3 API.
328 * See: "Components of Amazon S3" - API Version 2006-03-01 pg. 8
330 * Individual sub-resources are defined in several places. In the REST API,
331 * they they are represented by a "flag" in the "query string".
332 * See: "Constructing the CanonicalizedResource Element" - API Version 2006-03-01 pg. 60
334 * @param hdl: the S3Handle object
335 * @param verb: the HTTP request method
336 * @param bucket: the bucket to access, or NULL for none
337 * @param key: the key to access, or NULL for none
338 * @param subresource: the "sub-resource" to request (e.g. "acl") or NULL for none
339 * @param query: the query string to send (not including th initial '?'),
341 * @param read_func: the callback for reading data
342 * Will use s3_empty_read_func if NULL is passed in.
343 * @param read_reset_func: the callback for to reset reading data
344 * @param size_func: the callback to get the number of bytes to upload
345 * @param md5_func: the callback to get the MD5 hash of the data to upload
346 * @param read_data: pointer to pass to the above functions
347 * @param write_func: the callback for writing data.
348 * Will use s3_counter_write_func if NULL is passed in.
349 * @param write_reset_func: the callback for to reset writing data
350 * @param write_data: pointer to pass to C{write_func}
351 * @param progress_func: the callback for progress information
352 * @param progress_data: pointer to pass to C{progress_func}
353 * @param result_handling: instructions for handling the results; see above.
354 * @returns: the result specified by result_handling; details of the response
355 * are then available in C{hdl->last*}
358 perform_request(S3Handle *hdl,
362 const char *subresource,
364 s3_read_func read_func,
365 s3_reset_func read_reset_func,
366 s3_size_func size_func,
367 s3_md5_func md5_func,
369 s3_write_func write_func,
370 s3_reset_func write_reset_func,
372 s3_progress_func progress_func,
373 gpointer progress_data,
374 const result_handling_t *result_handling);
377 * a CURLOPT_WRITEFUNCTION to save part of the response in memory and
378 * call an external function if one was provided.
381 s3_internal_write_func(void *ptr, size_t size, size_t nmemb, void * stream);
384 * a function to reset to our internal buffer
387 s3_internal_reset_func(void * stream);
390 * a CURLOPT_HEADERFUNCTION to save the ETag header only.
393 s3_internal_header_func(void *ptr, size_t size, size_t nmemb, void * stream);
396 compile_regexes(void);
399 * Static function implementations
401 static s3_error_code_t
402 s3_error_code_from_name(char *s3_error_name)
406 if (!s3_error_name) return S3_ERROR_Unknown;
408 /* do a brute-force search through the list, since it's not sorted */
409 for (i = 0; i < S3_ERROR_END; i++) {
410 if (g_strcasecmp(s3_error_name, s3_error_code_names[i]) == 0)
414 return S3_ERROR_Unknown;
418 s3_error_name_from_code(s3_error_code_t s3_error_code)
420 if (s3_error_code >= S3_ERROR_END)
421 s3_error_code = S3_ERROR_Unknown;
423 return s3_error_code_names[s3_error_code];
427 s3_curl_supports_ssl(void)
429 static int supported = -1;
430 if (supported == -1) {
431 #if defined(CURL_VERSION_SSL)
432 curl_version_info_data *info = curl_version_info(CURLVERSION_NOW);
433 if (info->features & CURL_VERSION_SSL)
446 lookup_result(const result_handling_t *result_handling,
448 s3_error_code_t s3_error_code,
451 while (result_handling->response_code
452 || result_handling->s3_error_code
453 || result_handling->curl_code) {
454 if ((result_handling->response_code && result_handling->response_code != response_code)
455 || (result_handling->s3_error_code && result_handling->s3_error_code != s3_error_code)
456 || (result_handling->curl_code && result_handling->curl_code != curl_code)) {
461 return result_handling->result;
464 /* return the result for the terminator, as the default */
465 return result_handling->result;
469 build_url(const char *bucket,
471 const char *subresource,
473 gboolean use_subdomain,
477 char *esc_bucket = NULL, *esc_key = NULL;
480 url = g_string_new("http");
482 g_string_append(url, "s");
484 g_string_append(url, "://");
487 if (use_subdomain && bucket)
488 g_string_append_printf(url, "%s.s3.amazonaws.com/", bucket);
490 g_string_append(url, "s3.amazonaws.com/");
493 if (!use_subdomain && bucket) {
494 esc_bucket = curl_escape(bucket, 0);
495 if (!esc_bucket) goto cleanup;
496 g_string_append_printf(url, "%s", esc_bucket);
498 g_string_append(url, "/");
502 esc_key = curl_escape(key, 0);
503 if (!esc_key) goto cleanup;
504 g_string_append_printf(url, "%s", esc_key);
508 if (subresource || query)
509 g_string_append(url, "?");
512 g_string_append(url, subresource);
514 if (subresource && query)
515 g_string_append(url, "&");
518 g_string_append(url, query);
521 if (esc_bucket) curl_free(esc_bucket);
522 if (esc_key) curl_free(esc_key);
524 return g_string_free(url, FALSE);
527 static struct curl_slist *
528 authenticate_request(S3Handle *hdl,
532 const char *subresource,
533 const char *md5_hash,
534 gboolean use_subdomain)
541 GByteArray *md = NULL;
542 char *auth_base64 = NULL;
543 struct curl_slist *headers = NULL;
544 char *esc_bucket = NULL, *esc_key = NULL;
545 GString *auth_string = NULL;
547 /* Build the string to sign, per the S3 spec.
548 * See: "Authenticating REST Requests" - API Version 2006-03-01 pg 58
552 auth_string = g_string_new(verb);
553 g_string_append(auth_string, "\n");
555 /* Content-MD5 header */
557 g_string_append(auth_string, md5_hash);
558 g_string_append(auth_string, "\n");
560 /* Content-Type is empty*/
561 g_string_append(auth_string, "\n");
564 /* calculate the date */
567 if (!localtime_s(&tmp, &t)) g_debug("localtime error");
569 if (!localtime_r(&t, &tmp)) perror("localtime");
571 if (!strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tmp))
574 g_string_append(auth_string, date);
575 g_string_append(auth_string, "\n");
577 if (hdl->user_token) {
578 g_string_append(auth_string, AMAZON_SECURITY_HEADER);
579 g_string_append(auth_string, ":");
580 g_string_append(auth_string, hdl->user_token);
581 g_string_append(auth_string, ",");
582 g_string_append(auth_string, STS_PRODUCT_TOKEN);
583 g_string_append(auth_string, "\n");
586 /* CanonicalizedResource */
587 g_string_append(auth_string, "/");
590 g_string_append(auth_string, bucket);
592 esc_bucket = curl_escape(bucket, 0);
593 if (!esc_bucket) goto cleanup;
594 g_string_append(auth_string, esc_bucket);
598 if (bucket && (use_subdomain || key))
599 g_string_append(auth_string, "/");
602 esc_key = curl_escape(key, 0);
603 if (!esc_key) goto cleanup;
604 g_string_append(auth_string, esc_key);
608 g_string_append(auth_string, "?");
609 g_string_append(auth_string, subresource);
612 /* run HMAC-SHA1 on the canonicalized string */
613 md = g_byte_array_sized_new(EVP_MAX_MD_SIZE+1);
615 HMAC_Init_ex(&ctx, hdl->secret_key, (int) strlen(hdl->secret_key), EVP_sha1(), NULL);
616 HMAC_Update(&ctx, (unsigned char*) auth_string->str, auth_string->len);
617 HMAC_Final(&ctx, md->data, &md->len);
618 HMAC_CTX_cleanup(&ctx);
619 auth_base64 = s3_base64_encode(md);
621 /* append the new headers */
622 if (hdl->user_token) {
623 /* Devpay headers are included in hash. */
624 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", hdl->user_token);
625 headers = curl_slist_append(headers, buf);
628 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", STS_PRODUCT_TOKEN);
629 headers = curl_slist_append(headers, buf);
633 buf = g_strdup_printf("Authorization: AWS %s:%s",
634 hdl->access_key, auth_base64);
635 headers = curl_slist_append(headers, buf);
638 if (md5_hash && '\0' != md5_hash[0]) {
639 buf = g_strdup_printf("Content-MD5: %s", md5_hash);
640 headers = curl_slist_append(headers, buf);
644 buf = g_strdup_printf("Date: %s", date);
645 headers = curl_slist_append(headers, buf);
650 g_byte_array_free(md, TRUE);
652 g_string_free(auth_string, TRUE);
658 interpret_response(S3Handle *hdl,
660 char *curl_error_buffer,
664 const char *content_md5)
666 long response_code = 0;
667 regmatch_t pmatch[2];
668 char *error_name = NULL, *message = NULL;
669 char *body_copy = NULL;
672 if (!hdl) return FALSE;
674 if (hdl->last_message) g_free(hdl->last_message);
675 hdl->last_message = NULL;
677 /* bail out from a CURL error */
678 if (curl_code != CURLE_OK) {
679 hdl->last_curl_code = curl_code;
680 hdl->last_message = g_strdup_printf("CURL error: %s", curl_error_buffer);
684 /* CURL seems to think things were OK, so get its response code */
685 curl_easy_getinfo(hdl->curl, CURLINFO_RESPONSE_CODE, &response_code);
686 hdl->last_response_code = response_code;
688 /* check ETag, if present */
689 if (etag && content_md5 && 200 == response_code) {
690 if (etag && g_strcasecmp(etag, content_md5))
691 hdl->last_message = g_strdup("S3 Error: Possible data corruption (ETag returned by Amazon did not match the MD5 hash of the data sent)");
697 if (200 <= response_code && response_code < 400) {
698 /* 2xx and 3xx codes won't have a response body we care about */
699 hdl->last_s3_error_code = S3_ERROR_None;
703 /* Now look at the body to try to get the actual Amazon error message. Rather
704 * than parse out the XML, just use some regexes. */
706 /* impose a reasonable limit on body size */
707 if (body_len > MAX_ERROR_RESPONSE_LEN) {
708 hdl->last_message = g_strdup("S3 Error: Unknown (response body too large to parse)");
710 } else if (!body || body_len == 0) {
711 hdl->last_message = g_strdup("S3 Error: Unknown (empty response body)");
712 return TRUE; /* perhaps a network error; retry the request */
715 /* use strndup to get a zero-terminated string */
716 body_copy = g_strndup(body, body_len);
717 if (!body_copy) goto cleanup;
719 if (!s3_regexec_wrap(&error_name_regex, body_copy, 2, pmatch, 0))
720 error_name = find_regex_substring(body_copy, pmatch[1]);
722 if (!s3_regexec_wrap(&message_regex, body_copy, 2, pmatch, 0))
723 message = find_regex_substring(body_copy, pmatch[1]);
726 hdl->last_s3_error_code = s3_error_code_from_name(error_name);
730 hdl->last_message = message;
731 message = NULL; /* steal the reference to the string */
742 /* a CURLOPT_READFUNCTION to read data from a buffer. */
744 s3_buffer_read_func(void *ptr, size_t size, size_t nmemb, void * stream)
746 CurlBuffer *data = stream;
747 guint bytes_desired = (guint) size * nmemb;
749 /* check the number of bytes remaining, just to be safe */
750 if (bytes_desired > data->buffer_len - data->buffer_pos)
751 bytes_desired = data->buffer_len - data->buffer_pos;
753 memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
754 data->buffer_pos += bytes_desired;
756 return bytes_desired;
760 s3_buffer_size_func(void *stream)
762 CurlBuffer *data = stream;
763 return data->buffer_len;
767 s3_buffer_md5_func(void *stream)
769 CurlBuffer *data = stream;
770 GByteArray req_body_gba = {(guint8 *)data->buffer, data->buffer_len};
772 return s3_compute_md5_hash(&req_body_gba);
776 s3_buffer_reset_func(void *stream)
778 CurlBuffer *data = stream;
779 data->buffer_pos = 0;
782 /* a CURLOPT_WRITEFUNCTION to write data to a buffer. */
784 s3_buffer_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
786 CurlBuffer * data = stream;
787 guint new_bytes = (guint) size * nmemb;
788 guint bytes_needed = data->buffer_pos + new_bytes;
790 /* error out if the new size is greater than the maximum allowed */
791 if (data->max_buffer_size && bytes_needed > data->max_buffer_size)
794 /* reallocate if necessary. We use exponential sizing to make this
795 * happen less often. */
796 if (bytes_needed > data->buffer_len) {
797 guint new_size = MAX(bytes_needed, data->buffer_len * 2);
798 if (data->max_buffer_size) {
799 new_size = MIN(new_size, data->max_buffer_size);
801 data->buffer = g_realloc(data->buffer, new_size);
802 data->buffer_len = new_size;
805 return 0; /* returning zero signals an error to libcurl */
807 /* actually copy the data to the buffer */
808 memcpy(data->buffer + data->buffer_pos, ptr, new_bytes);
809 data->buffer_pos += new_bytes;
811 /* signal success to curl */
815 /* a CURLOPT_READFUNCTION that writes nothing. */
817 s3_empty_read_func(G_GNUC_UNUSED void *ptr, G_GNUC_UNUSED size_t size, G_GNUC_UNUSED size_t nmemb, G_GNUC_UNUSED void * stream)
823 s3_empty_size_func(G_GNUC_UNUSED void *stream)
829 s3_empty_md5_func(G_GNUC_UNUSED void *stream)
831 static const GByteArray empty = {(guint8 *) "", 0};
833 return s3_compute_md5_hash(&empty);
836 /* a CURLOPT_WRITEFUNCTION to write data that just counts data.
837 * s3_write_data should be NULL or a pointer to an gint64.
840 s3_counter_write_func(G_GNUC_UNUSED void *ptr, size_t size, size_t nmemb, void *stream)
842 gint64 *count = (gint64*) stream, inc = nmemb*size;
844 if (count) *count += inc;
849 s3_counter_reset_func(void *stream)
851 gint64 *count = (gint64*) stream;
853 if (count) *count = 0;
857 /* a CURLOPT_READFUNCTION to read data from a file. */
859 s3_file_read_func(void *ptr, size_t size, size_t nmemb, void * stream)
861 HANDLE *hFile = (HANDLE *) stream;
864 ReadFile(hFile, ptr, (DWORD) size*nmemb, &bytes_read, NULL);
869 s3_file_size_func(void *stream)
871 HANDLE *hFile = (HANDLE *) stream;
872 DWORD size = GetFileSize(hFile, NULL);
874 if (INVALID_FILE_SIZE == size) {
882 s3_file_md5_func(void *stream)
884 #define S3_MD5_BUF_SIZE (10*1024)
885 HANDLE *hFile = (HANDLE *) stream;
886 guint8 buf[S3_MD5_BUF_SIZE];
889 GByteArray *ret = NULL;
891 g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
893 ret = g_byte_array_sized_new(S3_MD5_HASH_BYTE_LEN);
894 g_byte_array_set_size(ret, S3_MD5_HASH_BYTE_LEN);
897 while (ReadFile(hFile, buf, S3_MD5_BUF_SIZE, &bytes_read, NULL)) {
898 MD5_Update(&md5_ctx, buf, bytes_read);
900 MD5_Final(ret->data, &md5_ctx);
902 g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
904 #undef S3_MD5_BUF_SIZE
908 s3_file_reset_func(void *stream)
910 g_assert(INVALID_SET_FILE_POINTER != SetFilePointer(hFile, 0, NULL, FILE_BEGIN));
913 /* a CURLOPT_WRITEFUNCTION to write data to a file. */
915 s3_file_write_func(void *ptr, size_t size, size_t nmemb, void *stream)
917 HANDLE *hFile = (HANDLE *) stream;
920 WriteFile(hFile, ptr, (DWORD) size*nmemb, &bytes_written, NULL);
921 return bytes_written;
926 curl_debug_message(CURL *curl G_GNUC_UNUSED,
930 void *unused G_GNUC_UNUSED)
934 char **lines, **line;
941 case CURLINFO_HEADER_IN:
942 lineprefix="Hdr In: ";
945 case CURLINFO_HEADER_OUT:
946 lineprefix="Hdr Out: ";
950 /* ignore data in/out -- nobody wants to see that in the
955 /* split the input into lines */
956 message = g_strndup(s, (gsize) len);
957 lines = g_strsplit(message, "\n", -1);
960 for (line = lines; *line; line++) {
961 if (**line == '\0') continue; /* skip blank lines */
962 g_debug("%s%s", lineprefix, *line);
970 perform_request(S3Handle *hdl,
974 const char *subresource,
976 s3_read_func read_func,
977 s3_reset_func read_reset_func,
978 s3_size_func size_func,
979 s3_md5_func md5_func,
981 s3_write_func write_func,
982 s3_reset_func write_reset_func,
984 s3_progress_func progress_func,
985 gpointer progress_data,
986 const result_handling_t *result_handling)
988 gboolean use_subdomain;
990 s3_result_t result = S3_RESULT_FAIL; /* assume the worst.. */
991 CURLcode curl_code = CURLE_OK;
992 char curl_error_buffer[CURL_ERROR_SIZE] = "";
993 struct curl_slist *headers = NULL;
994 S3InternalData int_writedata = {{NULL, 0, 0, MAX_ERROR_RESPONSE_LEN}, NULL, NULL, NULL, FALSE, NULL};
995 gboolean should_retry;
997 gulong backoff = EXPONENTIAL_BACKOFF_START_USEC;
998 /* corresponds to PUT, HEAD, GET, and POST */
999 int curlopt_upload = 0, curlopt_nobody = 0, curlopt_httpget = 0, curlopt_post = 0;
1000 /* do we want to examine the headers */
1001 const char *curlopt_customrequest = NULL;
1002 /* for MD5 calculation */
1003 GByteArray *md5_hash = NULL;
1004 gchar *md5_hash_hex = NULL, *md5_hash_b64 = NULL;
1005 size_t request_body_size = 0;
1007 g_assert(hdl != NULL && hdl->curl != NULL);
1011 use_subdomain = hdl->bucket_location? TRUE : FALSE;
1012 url = build_url(bucket, key, subresource, query, use_subdomain, hdl->use_ssl);
1013 if (!url) goto cleanup;
1015 /* libcurl may behave strangely if these are not set correctly */
1016 if (!strncmp(verb, "PUT", 4)) {
1018 } else if (!strncmp(verb, "GET", 4)) {
1019 curlopt_httpget = 1;
1020 } else if (!strncmp(verb, "POST", 5)) {
1022 } else if (!strncmp(verb, "HEAD", 5)) {
1025 curlopt_customrequest = verb;
1029 request_body_size = size_func(read_data);
1033 md5_hash = md5_func(read_data);
1035 md5_hash_b64 = s3_base64_encode(md5_hash);
1036 md5_hash_hex = s3_hex_encode(md5_hash);
1037 g_byte_array_free(md5_hash, TRUE);
1041 /* Curl will use fread() otherwise */
1042 read_func = s3_empty_read_func;
1046 int_writedata.write_func = write_func;
1047 int_writedata.reset_func = write_reset_func;
1048 int_writedata.write_data = write_data;
1050 /* Curl will use fwrite() otherwise */
1051 int_writedata.write_func = s3_counter_write_func;
1052 int_writedata.reset_func = s3_counter_reset_func;
1053 int_writedata.write_data = NULL;
1059 curl_slist_free_all(headers);
1061 curl_error_buffer[0] = '\0';
1062 if (read_reset_func) {
1063 read_reset_func(read_data);
1065 /* calls write_reset_func */
1066 s3_internal_reset_func(&int_writedata);
1068 /* set up the request */
1069 headers = authenticate_request(hdl, verb, bucket, key, subresource,
1070 md5_hash_b64, hdl->bucket_location? TRUE : FALSE);
1072 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_VERBOSE, hdl->verbose)))
1075 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_DEBUGFUNCTION,
1076 curl_debug_message)))
1079 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_ERRORBUFFER,
1080 curl_error_buffer)))
1082 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS, 1)))
1084 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_FOLLOWLOCATION, 1)))
1086 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_URL, url)))
1088 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPHEADER,
1091 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEFUNCTION, s3_internal_write_func)))
1093 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEDATA, &int_writedata)))
1095 /* Note: we always have to set this apparently, for consistent "end of header" detection */
1096 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HEADERFUNCTION, s3_internal_header_func)))
1098 /* Note: if set, CURLOPT_HEADERDATA seems to also be used for CURLOPT_WRITEDATA ? */
1099 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HEADERDATA, &int_writedata)))
1101 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_PROGRESSFUNCTION, progress_func)))
1103 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_PROGRESSDATA, progress_data)))
1106 #ifdef CURLOPT_INFILESIZE_LARGE
1107 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)request_body_size)))
1110 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE, (long)request_body_size)))
1114 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPGET, curlopt_httpget)))
1116 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, curlopt_upload)))
1118 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_POST, curlopt_post)))
1120 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOBODY, curlopt_nobody)))
1122 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CUSTOMREQUEST,
1123 curlopt_customrequest)))
1127 if (curlopt_upload) {
1128 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION, read_func)))
1130 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, read_data)))
1133 /* Clear request_body options. */
1134 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION,
1137 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA,
1142 /* Perform the request */
1143 curl_code = curl_easy_perform(hdl->curl);
1146 /* interpret the response into hdl->last* */
1147 curl_error: /* (label for short-circuiting the curl_easy_perform call) */
1148 should_retry = interpret_response(hdl, curl_code, curl_error_buffer,
1149 int_writedata.resp_buf.buffer, int_writedata.resp_buf.buffer_pos, int_writedata.etag, md5_hash_hex);
1151 /* and, unless we know we need to retry, see what we're to do now */
1152 if (!should_retry) {
1153 result = lookup_result(result_handling, hdl->last_response_code,
1154 hdl->last_s3_error_code, hdl->last_curl_code);
1156 /* break out of the while(1) unless we're retrying */
1157 if (result != S3_RESULT_RETRY)
1161 if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES) {
1162 /* we're out of retries, so annotate hdl->last_message appropriately and bail
1164 char *m = g_strdup_printf("Too many retries; last message was '%s'", hdl->last_message);
1165 if (hdl->last_message) g_free(hdl->last_message);
1166 hdl->last_message = m;
1167 result = S3_RESULT_FAIL;
1173 backoff *= EXPONENTIAL_BACKOFF_BASE;
1176 if (result != S3_RESULT_OK) {
1177 g_debug(_("%s %s failed with %d/%s"), verb, url,
1178 hdl->last_response_code,
1179 s3_error_name_from_code(hdl->last_s3_error_code));
1184 if (headers) curl_slist_free_all(headers);
1185 g_free(md5_hash_b64);
1186 g_free(md5_hash_hex);
1188 /* we don't deallocate the response body -- we keep it for later */
1189 hdl->last_response_body = int_writedata.resp_buf.buffer;
1190 hdl->last_response_body_size = int_writedata.resp_buf.buffer_pos;
1191 hdl->last_num_retries = retries;
1198 s3_internal_write_func(void *ptr, size_t size, size_t nmemb, void * stream)
1200 S3InternalData *data = (S3InternalData *) stream;
1203 if (!data->headers_done)
1206 bytes_saved = s3_buffer_write_func(ptr, size, nmemb, &data->resp_buf);
1207 if (data->write_func) {
1208 return data->write_func(ptr, size, nmemb, data->write_data);
1215 s3_internal_reset_func(void * stream)
1217 S3InternalData *data = (S3InternalData *) stream;
1219 s3_buffer_reset_func(&data->resp_buf);
1220 data->headers_done = FALSE;
1222 if (data->reset_func) {
1223 data->reset_func(data->write_data);
1228 s3_internal_header_func(void *ptr, size_t size, size_t nmemb, void * stream)
1230 static const char *final_header = "\r\n";
1232 regmatch_t pmatch[2];
1233 S3InternalData *data = (S3InternalData *) stream;
1235 header = g_strndup((gchar *) ptr, (gsize) size*nmemb);
1236 if (!s3_regexec_wrap(&etag_regex, header, 2, pmatch, 0))
1237 data->etag = find_regex_substring(header, pmatch[1]);
1238 if (!strcmp(final_header, header))
1239 data->headers_done = TRUE;
1245 compile_regexes(void)
1249 /* using POSIX regular expressions */
1250 struct {const char * str; int flags; regex_t *regex;} regexes[] = {
1251 {"<Code>[[:space:]]*([^<]*)[[:space:]]*</Code>", REG_EXTENDED | REG_ICASE, &error_name_regex},
1252 {"^ETag:[[:space:]]*\"([^\"]+)\"[[:space:]]*$", REG_EXTENDED | REG_ICASE | REG_NEWLINE, &etag_regex},
1253 {"<Message>[[:space:]]*([^<]*)[[:space:]]*</Message>", REG_EXTENDED | REG_ICASE, &message_regex},
1254 {"^[a-z0-9]((-*[a-z0-9])|(\\.[a-z0-9])){2,62}$", REG_EXTENDED | REG_NOSUB, &subdomain_regex},
1255 {"(/>)|(>([^<]*)</LocationConstraint>)", REG_EXTENDED | REG_ICASE, &location_con_regex},
1258 char regmessage[1024];
1262 for (i = 0; regexes[i].str; i++) {
1263 reg_result = regcomp(regexes[i].regex, regexes[i].str, regexes[i].flags);
1264 if (reg_result != 0) {
1265 size = regerror(reg_result, regexes[i].regex, regmessage, sizeof(regmessage));
1266 g_error(_("Regex error: %s"), regmessage);
1270 #else /* ! HAVE_REGEX_H */
1271 /* using PCRE via GLib */
1272 struct {const char * str; int flags; regex_t *regex;} regexes[] = {
1273 {"<Code>\\s*([^<]*)\\s*</Code>",
1274 G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
1276 {"^ETag:\\s*\"([^\"]+)\"\\s*$",
1277 G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
1279 {"<Message>\\s*([^<]*)\\s*</Message>",
1280 G_REGEX_OPTIMIZE | G_REGEX_CASELESS,
1282 {"^[a-z0-9]((-*[a-z0-9])|(\\.[a-z0-9])){2,62}$",
1283 G_REGEX_OPTIMIZE | G_REGEX_NO_AUTO_CAPTURE,
1285 {"(/>)|(>([^<]*)</LocationConstraint>)",
1287 &location_con_regex},
1293 for (i = 0; regexes[i].str; i++) {
1294 *(regexes[i].regex) = g_regex_new(regexes[i].str, regexes[i].flags, 0, &err);
1296 g_error(_("Regex error: %s"), err->message);
1306 * Public function implementations
1309 gboolean s3_init(void)
1311 static GStaticMutex mutex = G_STATIC_MUTEX_INIT;
1312 static gboolean init = FALSE, ret;
1314 /* n.b. curl_global_init is called in common-src/glib-util.c:glib_init() */
1316 g_static_mutex_lock (&mutex);
1318 ret = compile_regexes();
1321 g_static_mutex_unlock(&mutex);
1326 s3_curl_location_compat(void)
1328 curl_version_info_data *info;
1330 info = curl_version_info(CURLVERSION_NOW);
1331 return info->version_num > 0x070a02;
1335 s3_bucket_location_compat(const char *bucket)
1337 return !s3_regexec_wrap(&subdomain_regex, bucket, 0, NULL, 0);
1341 s3_open(const char *access_key,
1342 const char *secret_key,
1343 const char *user_token,
1344 const char *bucket_location
1348 hdl = g_new0(S3Handle, 1);
1349 if (!hdl) goto error;
1351 hdl->verbose = FALSE;
1352 hdl->use_ssl = s3_curl_supports_ssl();
1354 g_assert(access_key);
1355 hdl->access_key = g_strdup(access_key);
1356 g_assert(secret_key);
1357 hdl->secret_key = g_strdup(secret_key);
1359 hdl->user_token = g_strdup(user_token);
1362 hdl->bucket_location = g_strdup(bucket_location);
1364 hdl->curl = curl_easy_init();
1365 if (!hdl->curl) goto error;
1375 s3_free(S3Handle *hdl)
1380 g_free(hdl->access_key);
1381 g_free(hdl->secret_key);
1382 if (hdl->user_token) g_free(hdl->user_token);
1383 if (hdl->bucket_location) g_free(hdl->bucket_location);
1384 if (hdl->curl) curl_easy_cleanup(hdl->curl);
1391 s3_reset(S3Handle *hdl)
1394 /* We don't call curl_easy_reset here, because doing that in curl
1395 * < 7.16 blanks the default CA certificate path, and there's no way
1396 * to get it back. */
1397 if (hdl->last_message) {
1398 g_free(hdl->last_message);
1399 hdl->last_message = NULL;
1402 hdl->last_response_code = 0;
1403 hdl->last_curl_code = 0;
1404 hdl->last_s3_error_code = 0;
1405 hdl->last_num_retries = 0;
1407 if (hdl->last_response_body) {
1408 g_free(hdl->last_response_body);
1409 hdl->last_response_body = NULL;
1412 hdl->last_response_body_size = 0;
1417 s3_error(S3Handle *hdl,
1418 const char **message,
1419 guint *response_code,
1420 s3_error_code_t *s3_error_code,
1421 const char **s3_error_name,
1422 CURLcode *curl_code,
1426 if (message) *message = hdl->last_message;
1427 if (response_code) *response_code = hdl->last_response_code;
1428 if (s3_error_code) *s3_error_code = hdl->last_s3_error_code;
1429 if (s3_error_name) *s3_error_name = s3_error_name_from_code(hdl->last_s3_error_code);
1430 if (curl_code) *curl_code = hdl->last_curl_code;
1431 if (num_retries) *num_retries = hdl->last_num_retries;
1433 /* no hdl? return something coherent, anyway */
1434 if (message) *message = "NULL S3Handle";
1435 if (response_code) *response_code = 0;
1436 if (s3_error_code) *s3_error_code = 0;
1437 if (s3_error_name) *s3_error_name = NULL;
1438 if (curl_code) *curl_code = 0;
1439 if (num_retries) *num_retries = 0;
1444 s3_verbose(S3Handle *hdl, gboolean verbose)
1446 hdl->verbose = verbose;
1450 s3_use_ssl(S3Handle *hdl, gboolean use_ssl)
1452 gboolean ret = TRUE;
1453 if (use_ssl & !s3_curl_supports_ssl()) {
1456 hdl->use_ssl = use_ssl;
1462 s3_strerror(S3Handle *hdl)
1464 const char *message;
1465 guint response_code;
1466 const char *s3_error_name;
1470 char s3_info[256] = "";
1471 char response_info[16] = "";
1472 char curl_info[32] = "";
1473 char retries_info[32] = "";
1475 s3_error(hdl, &message, &response_code, NULL, &s3_error_name, &curl_code, &num_retries);
1478 message = "Unkonwn S3 error";
1480 g_snprintf(s3_info, sizeof(s3_info), " (%s)", s3_error_name);
1482 g_snprintf(response_info, sizeof(response_info), " (HTTP %d)", response_code);
1484 g_snprintf(curl_info, sizeof(curl_info), " (CURLcode %d)", curl_code);
1486 g_snprintf(retries_info, sizeof(retries_info), " (after %d retries)", num_retries);
1488 return g_strdup_printf("%s%s%s%s%s", message, s3_info, curl_info, response_info, retries_info);
1491 /* Perform an upload. When this function returns, KEY and
1492 * BUFFER remain the responsibility of the caller.
1494 * @param self: the s3 device
1495 * @param bucket: the bucket to which the upload should be made
1496 * @param key: the key to which the upload should be made
1497 * @param buffer: the data to be uploaded
1498 * @param buffer_len: the length of the data to upload
1499 * @returns: false if an error ocurred
1502 s3_upload(S3Handle *hdl,
1505 s3_read_func read_func,
1506 s3_reset_func reset_func,
1507 s3_size_func size_func,
1508 s3_md5_func md5_func,
1510 s3_progress_func progress_func,
1511 gpointer progress_data)
1513 s3_result_t result = S3_RESULT_FAIL;
1514 static result_handling_t result_handling[] = {
1515 { 200, 0, 0, S3_RESULT_OK },
1516 RESULT_HANDLING_ALWAYS_RETRY,
1517 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1520 g_assert(hdl != NULL);
1522 result = perform_request(hdl, "PUT", bucket, key, NULL, NULL,
1523 read_func, reset_func, size_func, md5_func, read_data,
1524 NULL, NULL, NULL, progress_func, progress_data,
1527 return result == S3_RESULT_OK;
1531 /* Private structure for our "thunk", which tracks where the user is in the list
1533 struct list_keys_thunk {
1534 GSList *filename_list; /* all pending filenames */
1536 gboolean in_contents; /* look for "key" entities in here */
1537 gboolean in_common_prefixes; /* look for "prefix" entities in here */
1539 gboolean is_truncated;
1548 /* Functions for a SAX parser to parse the XML from Amazon */
1551 list_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
1552 const gchar *element_name,
1553 const gchar **attribute_names G_GNUC_UNUSED,
1554 const gchar **attribute_values G_GNUC_UNUSED,
1556 GError **error G_GNUC_UNUSED)
1558 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1560 thunk->want_text = 0;
1561 if (g_strcasecmp(element_name, "contents") == 0) {
1562 thunk->in_contents = 1;
1563 } else if (g_strcasecmp(element_name, "commonprefixes") == 0) {
1564 thunk->in_common_prefixes = 1;
1565 } else if (g_strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1566 thunk->want_text = 1;
1567 } else if (g_strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1568 thunk->want_text = 1;
1569 } else if (g_strcasecmp(element_name, "istruncated")) {
1570 thunk->want_text = 1;
1571 } else if (g_strcasecmp(element_name, "nextmarker")) {
1572 thunk->want_text = 1;
1577 list_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
1578 const gchar *element_name,
1580 GError **error G_GNUC_UNUSED)
1582 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1584 if (g_strcasecmp(element_name, "contents") == 0) {
1585 thunk->in_contents = 0;
1586 } else if (g_strcasecmp(element_name, "commonprefixes") == 0) {
1587 thunk->in_common_prefixes = 0;
1588 } else if (g_strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1589 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1591 } else if (g_strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1592 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1594 } else if (g_strcasecmp(element_name, "istruncated") == 0) {
1595 if (thunk->text && g_strncasecmp(thunk->text, "false", 5) != 0)
1596 thunk->is_truncated = TRUE;
1597 } else if (g_strcasecmp(element_name, "nextmarker") == 0) {
1598 if (thunk->next_marker) g_free(thunk->next_marker);
1599 thunk->next_marker = thunk->text;
1605 list_text(GMarkupParseContext *context G_GNUC_UNUSED,
1609 GError **error G_GNUC_UNUSED)
1611 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1613 if (thunk->want_text) {
1614 if (thunk->text) g_free(thunk->text);
1615 thunk->text = g_strndup(text, text_len);
1619 /* Perform a fetch from S3; several fetches may be involved in a
1620 * single listing operation */
1622 list_fetch(S3Handle *hdl,
1625 const char *delimiter,
1627 const char *max_keys)
1629 s3_result_t result = S3_RESULT_FAIL;
1630 static result_handling_t result_handling[] = {
1631 { 200, 0, 0, S3_RESULT_OK },
1632 RESULT_HANDLING_ALWAYS_RETRY,
1633 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1635 const char* pos_parts[][2] = {
1637 {"delimiter", delimiter},
1639 {"make-keys", max_keys},
1645 gboolean have_prev_part = FALSE;
1647 /* loop over possible parts to build query string */
1648 query = g_string_new("");
1649 for (i = 0; pos_parts[i][0]; i++) {
1650 if (pos_parts[i][1]) {
1652 g_string_append(query, "&");
1654 have_prev_part = TRUE;
1655 esc_value = curl_escape(pos_parts[i][1], 0);
1656 g_string_append_printf(query, "%s=%s", pos_parts[i][0], esc_value);
1657 curl_free(esc_value);
1661 /* and perform the request on that URI */
1662 result = perform_request(hdl, "GET", bucket, NULL, NULL, query->str,
1663 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1666 if (query) g_string_free(query, TRUE);
1672 s3_list_keys(S3Handle *hdl,
1675 const char *delimiter,
1678 struct list_keys_thunk thunk;
1679 GMarkupParseContext *ctxt = NULL;
1680 static GMarkupParser parser = { list_start_element, list_end_element, list_text, NULL, NULL };
1682 s3_result_t result = S3_RESULT_FAIL;
1686 thunk.filename_list = NULL;
1688 thunk.next_marker = NULL;
1690 /* Loop until S3 has given us the entire picture */
1692 /* get some data from S3 */
1693 result = list_fetch(hdl, bucket, prefix, delimiter, thunk.next_marker, NULL);
1694 if (result != S3_RESULT_OK) goto cleanup;
1696 /* run the parser over it */
1697 thunk.in_contents = FALSE;
1698 thunk.in_common_prefixes = FALSE;
1699 thunk.is_truncated = FALSE;
1700 thunk.want_text = FALSE;
1702 ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
1704 if (!g_markup_parse_context_parse(ctxt, hdl->last_response_body,
1705 hdl->last_response_body_size, &err)) {
1706 if (hdl->last_message) g_free(hdl->last_message);
1707 hdl->last_message = g_strdup(err->message);
1708 result = S3_RESULT_FAIL;
1712 if (!g_markup_parse_context_end_parse(ctxt, &err)) {
1713 if (hdl->last_message) g_free(hdl->last_message);
1714 hdl->last_message = g_strdup(err->message);
1715 result = S3_RESULT_FAIL;
1719 g_markup_parse_context_free(ctxt);
1721 } while (thunk.next_marker);
1724 if (err) g_error_free(err);
1725 if (thunk.text) g_free(thunk.text);
1726 if (thunk.next_marker) g_free(thunk.next_marker);
1727 if (ctxt) g_markup_parse_context_free(ctxt);
1729 if (result != S3_RESULT_OK) {
1730 g_slist_free(thunk.filename_list);
1733 *list = thunk.filename_list;
1739 s3_read(S3Handle *hdl,
1742 s3_write_func write_func,
1743 s3_reset_func reset_func,
1744 gpointer write_data,
1745 s3_progress_func progress_func,
1746 gpointer progress_data)
1748 s3_result_t result = S3_RESULT_FAIL;
1749 static result_handling_t result_handling[] = {
1750 { 200, 0, 0, S3_RESULT_OK },
1751 RESULT_HANDLING_ALWAYS_RETRY,
1752 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1755 g_assert(hdl != NULL);
1756 g_assert(write_func != NULL);
1758 result = perform_request(hdl, "GET", bucket, key, NULL, NULL,
1759 NULL, NULL, NULL, NULL, NULL, write_func, reset_func, write_data,
1760 progress_func, progress_data, result_handling);
1762 return result == S3_RESULT_OK;
1766 s3_delete(S3Handle *hdl,
1770 s3_result_t result = S3_RESULT_FAIL;
1771 static result_handling_t result_handling[] = {
1772 { 204, 0, 0, S3_RESULT_OK },
1773 RESULT_HANDLING_ALWAYS_RETRY,
1774 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1777 g_assert(hdl != NULL);
1779 result = perform_request(hdl, "DELETE", bucket, key, NULL, NULL,
1780 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1783 return result == S3_RESULT_OK;
1787 s3_make_bucket(S3Handle *hdl,
1791 s3_result_t result = S3_RESULT_FAIL;
1792 static result_handling_t result_handling[] = {
1793 { 200, 0, 0, S3_RESULT_OK },
1794 RESULT_HANDLING_ALWAYS_RETRY,
1795 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1797 regmatch_t pmatch[4];
1798 char *loc_end_open, *loc_content;
1799 CurlBuffer buf = {NULL, 0, 0, 0}, *ptr = NULL;
1800 s3_read_func read_func = NULL;
1801 s3_reset_func reset_func = NULL;
1802 s3_md5_func md5_func = NULL;
1803 s3_size_func size_func = NULL;
1805 g_assert(hdl != NULL);
1807 if (hdl->bucket_location) {
1808 if (s3_bucket_location_compat(bucket)) {
1810 buf.buffer = g_strdup_printf(AMAZON_BUCKET_CONF_TEMPLATE, hdl->bucket_location);
1811 buf.buffer_len = (guint) strlen(body);
1813 buf.max_buffer_size = buf.buffer_len;
1814 read_func = s3_buffer_read_func;
1815 reset_func = s3_buffer_reset_func;
1816 size_func = s3_buffer_size_func;
1817 md5_func = s3_buffer_md5_func;
1819 hdl->last_message = g_strdup_printf(_(
1820 "Location constraint given for Amazon S3 bucket, "
1821 "but the bucket name (%s) is not usable as a subdomain."), bucket);
1826 result = perform_request(hdl, "PUT", bucket, NULL, NULL, NULL,
1827 read_func, reset_func, size_func, md5_func, ptr,
1828 NULL, NULL, NULL, NULL, NULL, result_handling);
1830 if (result == S3_RESULT_OK ||
1831 (hdl->bucket_location && result != S3_RESULT_OK
1832 && hdl->last_s3_error_code == S3_ERROR_BucketAlreadyOwnedByYou)) {
1833 /* verify the that the location constraint on the existing bucket matches
1834 * the one that's configured.
1836 result = perform_request(hdl, "GET", bucket, NULL, "location", NULL,
1837 NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
1838 NULL, NULL, result_handling);
1840 /* note that we can check only one of the three AND conditions above
1841 * and infer that the others are true
1843 if (result == S3_RESULT_OK && hdl->bucket_location) {
1844 /* return to the default state of failure */
1845 result = S3_RESULT_FAIL;
1847 if (body) g_free(body);
1848 /* use strndup to get a null-terminated string */
1849 body = g_strndup(hdl->last_response_body, hdl->last_response_body_size);
1850 if (!body) goto cleanup;
1852 if (!s3_regexec_wrap(&location_con_regex, body, 4, pmatch, 0)) {
1853 loc_end_open = find_regex_substring(body, pmatch[1]);
1854 loc_content = find_regex_substring(body, pmatch[3]);
1856 /* The case of an empty string is special because XML allows
1857 * "self-closing" tags
1859 if ('\0' == hdl->bucket_location[0] &&
1860 '/' != loc_end_open[0] && '\0' != hdl->bucket_location[0])
1861 hdl->last_message = _("An empty location constraint is "
1862 "configured, but the bucket has a non-empty location constraint");
1863 else if (strncmp(loc_content, hdl->bucket_location, strlen(hdl->bucket_location)))
1864 hdl->last_message = _("The location constraint configured "
1865 "does not match the constraint currently on the bucket");
1867 result = S3_RESULT_OK;
1869 hdl->last_message = _("Unexpected location response from Amazon S3");
1875 if (body) g_free(body);
1877 return result == S3_RESULT_OK;