2 * Copyright (c) 2005 Zmanda, Inc. All Rights Reserved.
4 * This library is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU Lesser General Public License version 2.1 as
6 * published by the Free Software Foundation.
8 * This library is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
13 * You should have received a copy of the GNU Lesser General Public License
14 * along with this library; if not, write to the Free Software Foundation,
15 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
18 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
22 * - Compute and send Content-MD5 header
23 * - check SSL certificate
24 * - collect speed statistics
29 #include <sys/types.h>
39 #include <curl/curl.h>
41 /* Constant renamed after version 7.10.7 */
42 #ifndef CURLINFO_RESPONSE_CODE
43 #define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
46 /* We don't need OpenSSL's kerberos support, and it's broken in
48 #define OPENSSL_NO_KRB5
50 #ifdef HAVE_OPENSSL_HMAC_H
51 # include <openssl/hmac.h>
53 # ifdef HAVE_CRYPTO_HMAC_H
54 # include <crypto/hmac.h>
62 #include <openssl/err.h>
63 #include <openssl/ssl.h>
66 * Constants / definitions
69 /* Maximum key length as specified in the S3 documentation
70 * (*excluding* null terminator) */
71 #define S3_MAX_KEY_LENGTH 1024
73 #if defined(LIBCURL_FEATURE_SSL) && defined(LIBCURL_PROTOCOL_HTTPS)
74 # define S3_URL "https://s3.amazonaws.com"
76 # define S3_URL "http://s3.amazonaws.com"
79 #define AMAZON_SECURITY_HEADER "x-amz-security-token"
81 /* parameters for exponential backoff in the face of retriable errors */
84 #define EXPONENTIAL_BACKOFF_START_USEC 10000
85 /* double at each retry */
86 #define EXPONENTIAL_BACKOFF_BASE 2
87 /* retry 15 times (for a total of about 5 minutes spent waiting) */
88 #define EXPONENTIAL_BACKOFF_MAX_RETRIES 5
90 /* general "reasonable size" parameters */
91 #define MAX_ERROR_RESPONSE_LEN (100*1024)
93 /* Results which should always be retried */
94 #define RESULT_HANDLING_ALWAYS_RETRY \
95 { 400, S3_ERROR_RequestTimeout, 0, S3_RESULT_RETRY }, \
96 { 409, S3_ERROR_OperationAborted, 0, S3_RESULT_RETRY }, \
97 { 412, S3_ERROR_PreconditionFailed, 0, S3_RESULT_RETRY }, \
98 { 500, S3_ERROR_InternalError, 0, S3_RESULT_RETRY }, \
99 { 501, S3_ERROR_NotImplemented, 0, S3_RESULT_RETRY }, \
100 { 0, 0, CURLE_COULDNT_CONNECT, S3_RESULT_RETRY }, \
101 { 0, 0, CURLE_PARTIAL_FILE, S3_RESULT_RETRY }, \
102 { 0, 0, CURLE_OPERATION_TIMEOUTED, S3_RESULT_RETRY }, \
103 { 0, 0, CURLE_SEND_ERROR, S3_RESULT_RETRY }, \
104 { 0, 0, CURLE_RECV_ERROR, S3_RESULT_RETRY }
107 * Data structures and associated functions
111 /* (all strings in this struct are freed by s3_free()) */
123 /* information from the last request */
125 guint last_response_code;
126 s3_error_code_t last_s3_error_code;
127 CURLcode last_curl_code;
128 guint last_num_retries;
129 void *last_response_body;
130 guint last_response_body_size;
136 /* (see preprocessor magic in s3.h) */
138 static char * s3_error_code_names[] = {
139 #define S3_ERROR(NAME) #NAME
144 /* Convert an s3 error name to an error code. This function
145 * matches strings case-insensitively, and is appropriate for use
146 * on data from the network.
148 * @param s3_error_code: the error name
149 * @returns: the error code (see constants in s3.h)
151 static s3_error_code_t
152 s3_error_code_from_name(char *s3_error_name);
154 /* Convert an s3 error code to a string
156 * @param s3_error_code: the error code to convert
157 * @returns: statically allocated string
160 s3_error_name_from_code(s3_error_code_t s3_error_code);
165 /* result handling is specified by a static array of result_handling structs,
166 * which match based on response_code (from HTTP) and S3 error code. The result
167 * given for the first match is used. 0 acts as a wildcard for both response_code
168 * and s3_error_code. The list is terminated with a struct containing 0 for both
169 * response_code and s3_error_code; the result for that struct is the default
172 * See RESULT_HANDLING_ALWAYS_RETRY for an example.
175 S3_RESULT_RETRY = -1,
180 typedef struct result_handling {
182 s3_error_code_t s3_error_code;
187 /* Lookup a result in C{result_handling}.
189 * @param result_handling: array of handling specifications
190 * @param response_code: response code from operation
191 * @param s3_error_code: s3 error code from operation, if any
192 * @param curl_code: the CURL error, if any
193 * @returns: the matching result
196 lookup_result(const result_handling_t *result_handling,
198 s3_error_code_t s3_error_code,
202 * Precompiled regular expressions */
204 static const char *error_name_regex_string = "<Code>[:space:]*([^<]*)[:space:]*</Code>";
205 static const char *message_regex_string = "<Message>[:space:]*([^<]*)[:space:]*</Message>";
206 static regex_t error_name_regex, message_regex;
212 /* Build a resource URI as /[bucket[/key]], with proper URL
215 * The caller is responsible for freeing the resulting string.
217 * @param bucket: the bucket, or NULL if none is involved
218 * @param key: the key within the bucket, or NULL if none is involved
219 * @returns: completed URI
222 build_resource(const char *bucket,
225 /* Create proper authorization headers for an Amazon S3 REST
226 * request to C{headers}.
228 * @note: C{X-Amz} headers (in C{headers}) must
230 * - be in alphabetical order
231 * - have no spaces around the colon
232 * (don't yell at me -- see the Amazon Developer Guide)
234 * @param hdl: the S3Handle object
235 * @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
236 * @param resource: the resource being accessed
238 static struct curl_slist *
239 authenticate_request(S3Handle *hdl,
241 const char *resource);
243 /* Interpret the response to an S3 operation, assuming CURL completed its request
244 * successfully. This function fills in the relevant C{hdl->last*} members.
246 * @param hdl: The S3Handle object
247 * @param body: the response body
248 * @param body_len: the length of the response body
249 * @returns: TRUE if the response should be retried (e.g., network error)
252 interpret_response(S3Handle *hdl,
254 char *curl_error_buffer,
258 /* Perform an S3 operation. This function handles all of the details
259 * of retryig requests and so on.
261 * @param hdl: the S3Handle object
262 * @param resource: the UTF-8 encoded resource to access
263 (without query parameters)
264 * @param uri: the urlencoded URI to access at Amazon (may be identical to resource)
265 * @param verb: the HTTP request method
266 * @param request_body: the request body, or NULL if none should be sent
267 * @param request_body_size: the length of the request body
268 * @param max_response_size: the maximum number of bytes to accept in the
269 * response, or 0 for no limit.
270 * @param preallocate_response_size: for more efficient operation, preallocate
271 * a buffer of this size for the response body. Addition space will be allocated
272 * if the response exceeds this size.
273 * @param result_handling: instructions for handling the results; see above.
274 * @returns: the result specified by result_handling; details of the response
275 * are then available in C{hdl->last*}
278 perform_request(S3Handle *hdl,
279 const char *resource,
282 const void *request_body,
283 guint request_body_size,
284 guint max_response_size,
285 guint preallocate_response_size,
286 const result_handling_t *result_handling);
289 * Static function implementations
292 /* {{{ s3_error_code_from_name */
293 static s3_error_code_t
294 s3_error_code_from_name(char *s3_error_name)
298 if (!s3_error_name) return S3_ERROR_Unknown;
300 /* do a brute-force search through the list, since it's not sorted */
301 for (i = 0; i < S3_ERROR_END; i++) {
302 if (strcasecmp(s3_error_name, s3_error_code_names[i]) == 0)
306 return S3_ERROR_Unknown;
310 /* {{{ s3_error_name_from_code */
312 s3_error_name_from_code(s3_error_code_t s3_error_code)
314 if (s3_error_code >= S3_ERROR_END)
315 s3_error_code = S3_ERROR_Unknown;
317 if (s3_error_code == 0)
320 return s3_error_code_names[s3_error_code];
324 /* {{{ lookup_result */
326 lookup_result(const result_handling_t *result_handling,
328 s3_error_code_t s3_error_code,
331 g_return_val_if_fail(result_handling != NULL, S3_RESULT_FAIL);
333 while (result_handling->response_code
334 || result_handling->s3_error_code
335 || result_handling->curl_code) {
336 if ((result_handling->response_code && result_handling->response_code != response_code)
337 || (result_handling->s3_error_code && result_handling->s3_error_code != s3_error_code)
338 || (result_handling->curl_code && result_handling->curl_code != curl_code)) {
343 return result_handling->result;
346 /* return the result for the terminator, as the default */
347 return result_handling->result;
351 /* {{{ build_resource */
353 build_resource(const char *bucket,
356 char *esc_bucket = NULL, *esc_key = NULL;
357 char *resource = NULL;
360 if (!(esc_bucket = curl_escape(bucket, 0)))
364 if (!(esc_key = curl_escape(key, 0)))
369 resource = g_strdup_printf("/%s/%s", esc_bucket, esc_key);
371 resource = g_strdup_printf("/%s", esc_bucket);
374 resource = g_strdup("/");
377 if (esc_bucket) curl_free(esc_bucket);
378 if (esc_key) curl_free(esc_key);
384 /* {{{ authenticate_request */
385 static struct curl_slist *
386 authenticate_request(S3Handle *hdl,
388 const char *resource)
395 char md_value[EVP_MAX_MD_SIZE+1];
396 char auth_base64[40];
398 struct curl_slist *headers = NULL;
401 /* calculate the date */
403 if (!localtime_r(&t, &tmp)) perror("localtime");
404 if (!strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tmp))
407 /* run HMAC-SHA1 on the canonicalized string */
409 HMAC_Init_ex(&ctx, hdl->secret_key, strlen(hdl->secret_key), EVP_sha1(), NULL);
410 auth_string = g_strconcat(verb, "\n\n\n", date, "\n",
412 AMAZON_SECURITY_HEADER, ":",
413 hdl->user_token, ",",
414 STS_PRODUCT_TOKEN, "\n",
417 HMAC_Update(&ctx, (unsigned char*) auth_string, strlen(auth_string));
419 md_len = EVP_MAX_MD_SIZE;
420 HMAC_Final(&ctx, (unsigned char*)md_value, &md_len);
421 HMAC_CTX_cleanup(&ctx);
422 base64_encode(md_value, md_len, auth_base64, sizeof(auth_base64));
424 /* append the new headers */
426 /* Devpay headers are included in hash. */
427 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", hdl->user_token);
428 headers = curl_slist_append(headers, buf);
431 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", STS_PRODUCT_TOKEN);
432 headers = curl_slist_append(headers, buf);
436 buf = g_strdup_printf("Authorization: AWS %s:%s",
437 hdl->access_key, auth_base64);
438 headers = curl_slist_append(headers, buf);
441 buf = g_strdup_printf("Date: %s", date);
442 headers = curl_slist_append(headers, buf);
449 /* {{{ interpret_response */
451 regex_error(regex_t *regex, int reg_result)
456 size = regerror(reg_result, regex, NULL, 0);
457 message = g_malloc(size);
458 if (!message) abort(); /* we're really out of luck */
459 regerror(reg_result, regex, message, size);
461 /* this is programmer error (bad regexp), so just log
462 * and abort(). There's no good way to signal a
463 * permanaent error from interpret_response. */
464 g_error(_("Regex error: %s"), message);
465 g_assert_not_reached();
469 interpret_response(S3Handle *hdl,
471 char *curl_error_buffer,
475 long response_code = 0;
476 regmatch_t pmatch[2];
478 char *error_name = NULL, *message = NULL;
479 char *body_copy = NULL;
481 if (!hdl) return FALSE;
483 if (hdl->last_message) g_free(hdl->last_message);
484 hdl->last_message = NULL;
486 /* bail out from a CURL error */
487 if (curl_code != CURLE_OK) {
488 hdl->last_curl_code = curl_code;
489 hdl->last_message = g_strdup_printf("CURL error: %s", curl_error_buffer);
493 /* CURL seems to think things were OK, so get its response code */
494 curl_easy_getinfo(hdl->curl, CURLINFO_RESPONSE_CODE, &response_code);
495 hdl->last_response_code = response_code;
497 /* 2xx and 3xx codes won't have a response body*/
498 if (200 <= response_code && response_code < 400) {
499 hdl->last_s3_error_code = S3_ERROR_None;
503 /* Now look at the body to try to get the actual Amazon error message. Rather
504 * than parse out the XML, just use some regexes. */
506 /* impose a reasonable limit on body size */
507 if (body_len > MAX_ERROR_RESPONSE_LEN) {
508 hdl->last_message = g_strdup("S3 Error: Unknown (response body too large to parse)");
510 } else if (!body || body_len == 0) {
511 hdl->last_message = g_strdup("S3 Error: Unknown (empty response body)");
512 return TRUE; /* perhaps a network error; retry the request */
515 /* use strndup to get a zero-terminated string */
516 body_copy = g_strndup(body, body_len);
517 if (!body_copy) goto cleanup;
519 reg_result = regexec(&error_name_regex, body_copy, 2, pmatch, 0);
520 if (reg_result != 0) {
521 if (reg_result == REG_NOMATCH) {
524 regex_error(&error_name_regex, reg_result);
525 g_assert_not_reached();
528 error_name = find_regex_substring(body_copy, pmatch[1]);
531 reg_result = regexec(&message_regex, body_copy, 2, pmatch, 0);
532 if (reg_result != 0) {
533 if (reg_result == REG_NOMATCH) {
536 regex_error(&message_regex, reg_result);
537 g_assert_not_reached();
540 message = find_regex_substring(body_copy, pmatch[1]);
544 hdl->last_s3_error_code = s3_error_code_from_name(error_name);
548 hdl->last_message = message;
549 message = NULL; /* steal the reference to the string */
553 if (body_copy) g_free(body_copy);
554 if (message) g_free(message);
555 if (error_name) g_free(error_name);
561 /* {{{ perform_request */
562 size_t buffer_readfunction(void *ptr, size_t size,
563 size_t nmemb, void * stream) {
564 CurlBuffer *data = stream;
565 guint bytes_desired = size * nmemb;
567 /* check the number of bytes remaining, just to be safe */
568 if (bytes_desired > data->buffer_len - data->buffer_pos)
569 bytes_desired = data->buffer_len - data->buffer_pos;
571 memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
572 data->buffer_pos += bytes_desired;
574 return bytes_desired;
578 buffer_writefunction(void *ptr, size_t size, size_t nmemb, void *stream)
580 CurlBuffer * data = stream;
581 guint new_bytes = size * nmemb;
582 guint bytes_needed = data->buffer_pos + new_bytes;
584 /* error out if the new size is greater than the maximum allowed */
585 if (data->max_buffer_size && bytes_needed > data->max_buffer_size)
588 /* reallocate if necessary. We use exponential sizing to make this
589 * happen less often. */
590 if (bytes_needed > data->buffer_len) {
591 guint new_size = MAX(bytes_needed, data->buffer_len * 2);
592 if (data->max_buffer_size) {
593 new_size = MIN(new_size, data->max_buffer_size);
595 data->buffer = g_realloc(data->buffer, new_size);
596 data->buffer_len = new_size;
598 g_return_val_if_fail(data->buffer, 0); /* returning zero signals an error to libcurl */
600 /* actually copy the data to the buffer */
601 memcpy(data->buffer + data->buffer_pos, ptr, new_bytes);
602 data->buffer_pos += new_bytes;
604 /* signal success to curl */
609 curl_debug_message(CURL *curl G_GNUC_UNUSED,
613 void *unused G_GNUC_UNUSED)
617 char **lines, **line;
624 case CURLINFO_HEADER_IN:
625 lineprefix="Hdr In: ";
628 case CURLINFO_HEADER_OUT:
629 lineprefix="Hdr Out: ";
633 /* ignore data in/out -- nobody wants to see that in the
638 /* split the input into lines */
639 message = g_strndup(s, len);
640 lines = g_strsplit(message, "\n", -1);
643 for (line = lines; *line; line++) {
644 if (**line == '\0') continue; /* skip blank lines */
645 g_debug("%s%s", lineprefix, *line);
653 perform_request(S3Handle *hdl,
654 const char *resource,
657 const void *request_body,
658 guint request_body_size,
659 guint max_response_size,
660 guint preallocate_response_size,
661 const result_handling_t *result_handling)
664 s3_result_t result = S3_RESULT_FAIL; /* assume the worst.. */
665 CURLcode curl_code = CURLE_OK;
666 char curl_error_buffer[CURL_ERROR_SIZE] = "";
667 struct curl_slist *headers = NULL;
668 CurlBuffer readdata = { (void*)request_body, request_body_size, 0, 0 };
669 CurlBuffer writedata = { NULL, 0, 0, max_response_size };
670 gboolean should_retry;
672 gulong backoff = EXPONENTIAL_BACKOFF_START_USEC;
674 g_return_val_if_fail(hdl != NULL && hdl->curl != NULL, S3_RESULT_FAIL);
678 url = g_strconcat(S3_URL, uri, NULL);
679 if (!url) goto cleanup;
681 if (preallocate_response_size) {
682 writedata.buffer = g_malloc(preallocate_response_size);
683 if (!writedata.buffer) goto cleanup;
684 writedata.buffer_len = preallocate_response_size;
690 curl_slist_free_all(headers);
692 readdata.buffer_pos = 0;
693 writedata.buffer_pos = 0;
694 curl_error_buffer[0] = '\0';
696 /* set up the request */
697 headers = authenticate_request(hdl, verb, resource);
699 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_VERBOSE, hdl->verbose)))
702 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_DEBUGFUNCTION,
703 curl_debug_message)))
705 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_ERRORBUFFER,
708 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS, 1)))
710 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_URL, url)))
712 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPHEADER,
715 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CUSTOMREQUEST,
718 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEFUNCTION, buffer_writefunction)))
720 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEDATA, &writedata)))
722 if (max_response_size) {
723 #ifdef CURLOPT_MAXFILESIZE_LARGE
724 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)max_response_size)))
727 # ifdef CURLOPT_MAXFILESIZE
728 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE, (long)max_response_size)))
731 /* no MAXFILESIZE option -- that's OK */
737 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 1)))
739 #ifdef CURLOPT_INFILESIZE_LARGE
740 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)request_body_size)))
743 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE, (long)request_body_size)))
746 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION, buffer_readfunction)))
748 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, &readdata)))
751 /* Clear request_body options. */
752 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 0)))
754 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION,
757 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA,
762 /* Perform the request */
763 curl_code = curl_easy_perform(hdl->curl);
766 /* interpret the response into hdl->last* */
767 curl_error: /* (label for short-circuiting the curl_easy_perform call) */
768 should_retry = interpret_response(hdl, curl_code, curl_error_buffer,
769 writedata.buffer, writedata.buffer_pos);
771 /* and, unless we know we need to retry, see what we're to do now */
773 result = lookup_result(result_handling, hdl->last_response_code,
774 hdl->last_s3_error_code, hdl->last_curl_code);
776 /* break out of the while(1) unless we're retrying */
777 if (result != S3_RESULT_RETRY)
781 if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES) {
782 /* we're out of retries, so annotate hdl->last_message appropriately and bail
784 char *m = g_strdup_printf("Too many retries; last message was '%s'", hdl->last_message);
785 if (hdl->last_message) g_free(hdl->last_message);
786 hdl->last_message = m;
787 result = S3_RESULT_FAIL;
793 backoff *= EXPONENTIAL_BACKOFF_BASE;
796 if (result != S3_RESULT_OK) {
797 g_debug(_("%s %s failed with %d/%s"), verb, url,
798 hdl->last_response_code,
799 s3_error_name_from_code(hdl->last_s3_error_code));
803 if (url) g_free(url);
804 if (headers) curl_slist_free_all(headers);
806 /* we don't deallocate the response body -- we keep it for later */
807 hdl->last_response_body = writedata.buffer;
808 hdl->last_response_body_size = writedata.buffer_pos;
809 hdl->last_num_retries = retries;
816 * Public function implementations
823 char regmessage[1024];
827 reg_result = regcomp(&error_name_regex, error_name_regex_string, REG_EXTENDED | REG_ICASE);
828 if (reg_result != 0) {
829 size = regerror(reg_result, &error_name_regex, regmessage, sizeof(regmessage));
830 g_error(_("Regex error: %s"), regmessage);
834 reg_result = regcomp(&message_regex, message_regex_string, REG_EXTENDED | REG_ICASE);
835 if (reg_result != 0) {
836 size = regerror(reg_result, &message_regex, regmessage, sizeof(regmessage));
837 g_error(_("Regex error: %s"), regmessage);
847 s3_open(const char *access_key,
848 const char *secret_key
851 const char *user_token
856 hdl = g_new0(S3Handle, 1);
857 if (!hdl) goto error;
859 hdl->verbose = FALSE;
861 hdl->access_key = g_strdup(access_key);
862 if (!hdl->access_key) goto error;
864 hdl->secret_key = g_strdup(secret_key);
865 if (!hdl->secret_key) goto error;
868 hdl->user_token = g_strdup(user_token);
869 if (!hdl->user_token) goto error;
872 hdl->curl = curl_easy_init();
873 if (!hdl->curl) goto error;
885 s3_free(S3Handle *hdl)
890 if (hdl->access_key) g_free(hdl->access_key);
891 if (hdl->secret_key) g_free(hdl->secret_key);
893 if (hdl->user_token) g_free(hdl->user_token);
895 if (hdl->curl) curl_easy_cleanup(hdl->curl);
904 s3_reset(S3Handle *hdl)
907 /* We don't call curl_easy_reset here, because doing that in curl
908 * < 7.16 blanks the default CA certificate path, and there's no way
910 if (hdl->last_message) {
911 g_free(hdl->last_message);
912 hdl->last_message = NULL;
915 hdl->last_response_code = 0;
916 hdl->last_curl_code = 0;
917 hdl->last_s3_error_code = 0;
918 hdl->last_num_retries = 0;
920 if (hdl->last_response_body) {
921 g_free(hdl->last_response_body);
922 hdl->last_response_body = NULL;
925 hdl->last_response_body_size = 0;
932 s3_error(S3Handle *hdl,
933 const char **message,
934 guint *response_code,
935 s3_error_code_t *s3_error_code,
936 const char **s3_error_name,
941 if (message) *message = hdl->last_message;
942 if (response_code) *response_code = hdl->last_response_code;
943 if (s3_error_code) *s3_error_code = hdl->last_s3_error_code;
944 if (s3_error_name) *s3_error_name = s3_error_name_from_code(hdl->last_s3_error_code);
945 if (curl_code) *curl_code = hdl->last_curl_code;
946 if (num_retries) *num_retries = hdl->last_num_retries;
948 /* no hdl? return something coherent, anyway */
949 if (message) *message = "NULL S3Handle";
950 if (response_code) *response_code = 0;
951 if (s3_error_code) *s3_error_code = 0;
952 if (s3_error_name) *s3_error_name = NULL;
953 if (curl_code) *curl_code = 0;
954 if (num_retries) *num_retries = 0;
961 s3_verbose(S3Handle *hdl, gboolean verbose)
963 hdl->verbose = verbose;
969 s3_strerror(S3Handle *hdl)
973 const char *s3_error_name;
977 char s3_info[256] = "";
978 char response_info[16] = "";
979 char curl_info[32] = "";
980 char retries_info[32] = "";
982 s3_error(hdl, &message, &response_code, NULL, &s3_error_name, &curl_code, &num_retries);
985 message = "Unkonwn S3 error";
987 g_snprintf(s3_info, sizeof(s3_info), " (%s)", s3_error_name);
989 g_snprintf(response_info, sizeof(response_info), " (HTTP %d)", response_code);
991 g_snprintf(curl_info, sizeof(curl_info), " (CURLcode %d)", curl_code);
993 g_snprintf(retries_info, sizeof(retries_info), " (after %d retries)", num_retries);
995 return g_strdup_printf("%s%s%s%s%s", message, s3_info, curl_info, response_info, retries_info);
1000 /* Perform an upload. When this function returns, KEY and
1001 * BUFFER remain the responsibility of the caller.
1003 * @param self: the s3 device
1004 * @param key: the key to which the upload should be made
1005 * @param buffer: the data to be uploaded
1006 * @param buffer_len: the length of the data to upload
1007 * @returns: false if an error ocurred
1010 s3_upload(S3Handle *hdl,
1016 char *resource = NULL;
1017 s3_result_t result = S3_RESULT_FAIL;
1018 static result_handling_t result_handling[] = {
1019 { 200, 0, 0, S3_RESULT_OK },
1020 RESULT_HANDLING_ALWAYS_RETRY,
1021 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1024 g_return_val_if_fail(hdl != NULL, FALSE);
1026 resource = build_resource(bucket, key);
1028 result = perform_request(hdl, resource, resource, "PUT",
1029 buffer, buffer_len, MAX_ERROR_RESPONSE_LEN, 0,
1034 return result == S3_RESULT_OK;
1038 /* {{{ s3_list_keys */
1040 /* Private structure for our "thunk", which tracks where the user is in the list
1042 struct list_keys_thunk {
1043 GSList *filename_list; /* all pending filenames */
1045 gboolean in_contents; /* look for "key" entities in here */
1046 gboolean in_common_prefixes; /* look for "prefix" entities in here */
1048 gboolean is_truncated;
1057 /* Functions for a SAX parser to parse the XML from Amazon */
1060 list_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
1061 const gchar *element_name,
1062 const gchar **attribute_names G_GNUC_UNUSED,
1063 const gchar **attribute_values G_GNUC_UNUSED,
1065 GError **error G_GNUC_UNUSED)
1067 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1069 thunk->want_text = 0;
1070 if (strcasecmp(element_name, "contents") == 0) {
1071 thunk->in_contents = 1;
1072 } else if (strcasecmp(element_name, "commonprefixes") == 0) {
1073 thunk->in_common_prefixes = 1;
1074 } else if (strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1075 thunk->want_text = 1;
1076 } else if (strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1077 thunk->want_text = 1;
1078 } else if (strcasecmp(element_name, "istruncated")) {
1079 thunk->want_text = 1;
1080 } else if (strcasecmp(element_name, "nextmarker")) {
1081 thunk->want_text = 1;
1086 list_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
1087 const gchar *element_name,
1089 GError **error G_GNUC_UNUSED)
1091 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1093 if (strcasecmp(element_name, "contents") == 0) {
1094 thunk->in_contents = 0;
1095 } else if (strcasecmp(element_name, "commonprefixes") == 0) {
1096 thunk->in_common_prefixes = 0;
1097 } else if (strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1098 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1100 } else if (strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1101 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1103 } else if (strcasecmp(element_name, "istruncated") == 0) {
1104 if (thunk->text && strncasecmp(thunk->text, "false", 5) != 0)
1105 thunk->is_truncated = TRUE;
1106 } else if (strcasecmp(element_name, "nextmarker") == 0) {
1107 if (thunk->next_marker) g_free(thunk->next_marker);
1108 thunk->next_marker = thunk->text;
1114 list_text(GMarkupParseContext *context G_GNUC_UNUSED,
1118 GError **error G_GNUC_UNUSED)
1120 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1122 if (thunk->want_text) {
1123 if (thunk->text) g_free(thunk->text);
1124 thunk->text = g_strndup(text, text_len);
1128 /* Helper function for list_fetch */
1130 list_build_url_component(char **rv,
1135 char *esc_value = NULL;
1136 char *new_rv = NULL;
1138 esc_value = curl_escape(value, 0);
1139 if (!esc_value) goto cleanup;
1141 new_rv = g_strconcat(*rv, delim, key, "=", esc_value, NULL);
1142 if (!new_rv) goto cleanup;
1146 curl_free(esc_value);
1151 if (new_rv) g_free(new_rv);
1152 if (esc_value) curl_free(esc_value);
1157 /* Perform a fetch from S3; several fetches may be involved in a
1158 * single listing operation */
1160 list_fetch(S3Handle *hdl,
1161 const char *resource,
1163 const char *delimiter,
1165 const char *max_keys)
1167 char *urldelim = "?";
1168 char *uri = g_strdup(resource);
1169 s3_result_t result = S3_RESULT_FAIL;
1170 static result_handling_t result_handling[] = {
1171 { 200, 0, 0, S3_RESULT_OK },
1172 RESULT_HANDLING_ALWAYS_RETRY,
1173 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1178 if (!list_build_url_component(&uri, urldelim, "prefix", prefix)) goto cleanup;
1182 if (!list_build_url_component(&uri, urldelim, "delimiter", delimiter)) goto cleanup;
1186 if (!list_build_url_component(&uri, urldelim, "marker", marker)) goto cleanup;
1190 if (!list_build_url_component(&uri, urldelim, "max-keys", max_keys)) goto cleanup;
1194 /* and perform the request on that URI */
1195 result = perform_request(hdl, resource, uri, "GET", NULL,
1196 0, MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1199 if (uri) g_free(uri);
1204 s3_list_keys(S3Handle *hdl,
1207 const char *delimiter,
1210 char *resource = NULL;
1211 struct list_keys_thunk thunk;
1212 GMarkupParseContext *ctxt = NULL;
1213 static GMarkupParser parser = { list_start_element, list_end_element, list_text, NULL, NULL };
1215 s3_result_t result = S3_RESULT_FAIL;
1219 thunk.filename_list = NULL;
1221 thunk.next_marker = NULL;
1223 resource = build_resource(bucket, NULL);
1224 if (!resource) goto cleanup;
1226 /* Loop until S3 has given us the entire picture */
1228 /* get some data from S3 */
1229 result = list_fetch(hdl, resource, prefix, delimiter, thunk.next_marker, NULL);
1230 if (result != S3_RESULT_OK) goto cleanup;
1232 /* run the parser over it */
1233 thunk.in_contents = FALSE;
1234 thunk.in_common_prefixes = FALSE;
1235 thunk.is_truncated = FALSE;
1236 thunk.want_text = FALSE;
1238 ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
1240 if (!g_markup_parse_context_parse(ctxt, hdl->last_response_body,
1241 hdl->last_response_body_size, &err)) {
1242 if (hdl->last_message) g_free(hdl->last_message);
1243 hdl->last_message = g_strdup(err->message);
1244 result = S3_RESULT_FAIL;
1248 if (!g_markup_parse_context_end_parse(ctxt, &err)) {
1249 if (hdl->last_message) g_free(hdl->last_message);
1250 hdl->last_message = g_strdup(err->message);
1251 result = S3_RESULT_FAIL;
1255 g_markup_parse_context_free(ctxt);
1257 } while (thunk.next_marker);
1260 if (err) g_error_free(err);
1261 if (thunk.text) g_free(thunk.text);
1262 if (thunk.next_marker) g_free(thunk.next_marker);
1263 if (resource) g_free(resource);
1264 if (ctxt) g_markup_parse_context_free(ctxt);
1266 if (result != S3_RESULT_OK) {
1267 g_slist_free(thunk.filename_list);
1270 *list = thunk.filename_list;
1278 s3_read(S3Handle *hdl,
1285 char *resource = NULL;
1286 s3_result_t result = S3_RESULT_FAIL;
1287 static result_handling_t result_handling[] = {
1288 { 200, 0, 0, S3_RESULT_OK },
1289 RESULT_HANDLING_ALWAYS_RETRY,
1290 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1293 g_return_val_if_fail(hdl != NULL, FALSE);
1294 g_assert(buf_ptr != NULL);
1295 g_assert(buf_size != NULL);
1300 resource = build_resource(bucket, key);
1302 result = perform_request(hdl, resource, resource,
1303 "GET", NULL, 0, max_size, 0, result_handling);
1306 /* copy the pointer to the result parameters and remove
1307 * our reference to it */
1308 if (result == S3_RESULT_OK) {
1309 *buf_ptr = hdl->last_response_body;
1310 *buf_size = hdl->last_response_body_size;
1312 hdl->last_response_body = NULL;
1313 hdl->last_response_body_size = 0;
1317 return result == S3_RESULT_OK;
1323 s3_delete(S3Handle *hdl,
1327 char *resource = NULL;
1328 s3_result_t result = S3_RESULT_FAIL;
1329 static result_handling_t result_handling[] = {
1330 { 204, 0, 0, S3_RESULT_OK },
1331 RESULT_HANDLING_ALWAYS_RETRY,
1332 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1335 g_return_val_if_fail(hdl != NULL, FALSE);
1337 resource = build_resource(bucket, key);
1339 result = perform_request(hdl, resource, resource, "DELETE", NULL, 0,
1340 MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1344 return result == S3_RESULT_OK;
1348 /* {{{ s3_make_bucket */
1350 s3_make_bucket(S3Handle *hdl,
1353 char *resource = NULL;
1354 s3_result_t result = result = S3_RESULT_FAIL;
1355 static result_handling_t result_handling[] = {
1356 { 200, 0, 0, S3_RESULT_OK },
1357 RESULT_HANDLING_ALWAYS_RETRY,
1358 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1361 g_return_val_if_fail(hdl != NULL, FALSE);
1363 resource = build_resource(bucket, NULL);
1365 result = perform_request(hdl, resource, resource, "PUT", NULL, 0,
1366 MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1370 return result == S3_RESULT_OK;