2 * Copyright (c) 2005 Zmanda, Inc. All Rights Reserved.
4 * This library is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU Lesser General Public License version 2.1 as
6 * published by the Free Software Foundation.
8 * This library is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
11 * License for more details.
13 * You should have received a copy of the GNU Lesser General Public License
14 * along with this library; if not, write to the Free Software Foundation,
15 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
17 * Contact information: Zmanda Inc., 505 N Mathlida Ave, Suite 120
18 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
22 * - Compute and send Content-MD5 header
23 * - check SSL certificate
24 * - collect speed statistics
29 #include <sys/types.h>
39 #include <curl/curl.h>
41 /* Constant renamed after version 7.10.7 */
42 #ifndef CURLINFO_RESPONSE_CODE
43 #define CURLINFO_RESPONSE_CODE CURLINFO_HTTP_CODE
46 /* We don't need OpenSSL's kerberos support, and it's broken in
48 #define OPENSSL_NO_KRB5
50 #ifdef HAVE_OPENSSL_HMAC_H
51 # include <openssl/hmac.h>
53 # ifdef HAVE_CRYPTO_HMAC_H
54 # include <crypto/hmac.h>
62 #include <openssl/err.h>
63 #include <openssl/ssl.h>
66 * Constants / definitions
69 /* Maximum key length as specified in the S3 documentation
70 * (*excluding* null terminator) */
71 #define S3_MAX_KEY_LENGTH 1024
73 #define AMAZON_SECURITY_HEADER "x-amz-security-token"
75 /* parameters for exponential backoff in the face of retriable errors */
78 #define EXPONENTIAL_BACKOFF_START_USEC 10000
79 /* double at each retry */
80 #define EXPONENTIAL_BACKOFF_BASE 2
81 /* retry 15 times (for a total of about 5 minutes spent waiting) */
82 #define EXPONENTIAL_BACKOFF_MAX_RETRIES 5
84 /* general "reasonable size" parameters */
85 #define MAX_ERROR_RESPONSE_LEN (100*1024)
87 /* Results which should always be retried */
88 #define RESULT_HANDLING_ALWAYS_RETRY \
89 { 400, S3_ERROR_RequestTimeout, 0, S3_RESULT_RETRY }, \
90 { 409, S3_ERROR_OperationAborted, 0, S3_RESULT_RETRY }, \
91 { 412, S3_ERROR_PreconditionFailed, 0, S3_RESULT_RETRY }, \
92 { 500, S3_ERROR_InternalError, 0, S3_RESULT_RETRY }, \
93 { 501, S3_ERROR_NotImplemented, 0, S3_RESULT_RETRY }, \
94 { 0, 0, CURLE_COULDNT_CONNECT, S3_RESULT_RETRY }, \
95 { 0, 0, CURLE_PARTIAL_FILE, S3_RESULT_RETRY }, \
96 { 0, 0, CURLE_OPERATION_TIMEOUTED, S3_RESULT_RETRY }, \
97 { 0, 0, CURLE_SEND_ERROR, S3_RESULT_RETRY }, \
98 { 0, 0, CURLE_RECV_ERROR, S3_RESULT_RETRY }
101 * Data structures and associated functions
105 /* (all strings in this struct are freed by s3_free()) */
117 /* information from the last request */
119 guint last_response_code;
120 s3_error_code_t last_s3_error_code;
121 CURLcode last_curl_code;
122 guint last_num_retries;
123 void *last_response_body;
124 guint last_response_body_size;
130 /* (see preprocessor magic in s3.h) */
132 static char * s3_error_code_names[] = {
133 #define S3_ERROR(NAME) #NAME
138 /* Convert an s3 error name to an error code. This function
139 * matches strings case-insensitively, and is appropriate for use
140 * on data from the network.
142 * @param s3_error_code: the error name
143 * @returns: the error code (see constants in s3.h)
145 static s3_error_code_t
146 s3_error_code_from_name(char *s3_error_name);
148 /* Convert an s3 error code to a string
150 * @param s3_error_code: the error code to convert
151 * @returns: statically allocated string
154 s3_error_name_from_code(s3_error_code_t s3_error_code);
156 /* Does this install of curl support SSL?
161 s3_curl_supports_ssl(void);
166 /* result handling is specified by a static array of result_handling structs,
167 * which match based on response_code (from HTTP) and S3 error code. The result
168 * given for the first match is used. 0 acts as a wildcard for both response_code
169 * and s3_error_code. The list is terminated with a struct containing 0 for both
170 * response_code and s3_error_code; the result for that struct is the default
173 * See RESULT_HANDLING_ALWAYS_RETRY for an example.
176 S3_RESULT_RETRY = -1,
181 typedef struct result_handling {
183 s3_error_code_t s3_error_code;
188 /* Lookup a result in C{result_handling}.
190 * @param result_handling: array of handling specifications
191 * @param response_code: response code from operation
192 * @param s3_error_code: s3 error code from operation, if any
193 * @param curl_code: the CURL error, if any
194 * @returns: the matching result
197 lookup_result(const result_handling_t *result_handling,
199 s3_error_code_t s3_error_code,
203 * Precompiled regular expressions */
205 static const char *error_name_regex_string = "<Code>[:space:]*([^<]*)[:space:]*</Code>";
206 static const char *message_regex_string = "<Message>[:space:]*([^<]*)[:space:]*</Message>";
207 static regex_t error_name_regex, message_regex;
213 /* Build a resource URI as /[bucket[/key]], with proper URL
216 * The caller is responsible for freeing the resulting string.
218 * @param bucket: the bucket, or NULL if none is involved
219 * @param key: the key within the bucket, or NULL if none is involved
220 * @returns: completed URI
223 build_resource(const char *bucket,
226 /* Create proper authorization headers for an Amazon S3 REST
227 * request to C{headers}.
229 * @note: C{X-Amz} headers (in C{headers}) must
231 * - be in alphabetical order
232 * - have no spaces around the colon
233 * (don't yell at me -- see the Amazon Developer Guide)
235 * @param hdl: the S3Handle object
236 * @param verb: capitalized verb for this request ('PUT', 'GET', etc.)
237 * @param resource: the resource being accessed
239 static struct curl_slist *
240 authenticate_request(S3Handle *hdl,
242 const char *resource);
244 /* Interpret the response to an S3 operation, assuming CURL completed its request
245 * successfully. This function fills in the relevant C{hdl->last*} members.
247 * @param hdl: The S3Handle object
248 * @param body: the response body
249 * @param body_len: the length of the response body
250 * @returns: TRUE if the response should be retried (e.g., network error)
253 interpret_response(S3Handle *hdl,
255 char *curl_error_buffer,
259 /* Perform an S3 operation. This function handles all of the details
260 * of retryig requests and so on.
262 * @param hdl: the S3Handle object
263 * @param resource: the UTF-8 encoded resource to access
264 (without query parameters)
265 * @param uri: the urlencoded URI to access at Amazon (may be identical to resource)
266 * @param verb: the HTTP request method
267 * @param request_body: the request body, or NULL if none should be sent
268 * @param request_body_size: the length of the request body
269 * @param max_response_size: the maximum number of bytes to accept in the
270 * response, or 0 for no limit.
271 * @param preallocate_response_size: for more efficient operation, preallocate
272 * a buffer of this size for the response body. Addition space will be allocated
273 * if the response exceeds this size.
274 * @param result_handling: instructions for handling the results; see above.
275 * @returns: the result specified by result_handling; details of the response
276 * are then available in C{hdl->last*}
279 perform_request(S3Handle *hdl,
280 const char *resource,
283 const void *request_body,
284 guint request_body_size,
285 guint max_response_size,
286 guint preallocate_response_size,
287 const result_handling_t *result_handling);
290 * Static function implementations
293 /* {{{ s3_error_code_from_name */
294 static s3_error_code_t
295 s3_error_code_from_name(char *s3_error_name)
299 if (!s3_error_name) return S3_ERROR_Unknown;
301 /* do a brute-force search through the list, since it's not sorted */
302 for (i = 0; i < S3_ERROR_END; i++) {
303 if (strcasecmp(s3_error_name, s3_error_code_names[i]) == 0)
307 return S3_ERROR_Unknown;
311 /* {{{ s3_error_name_from_code */
313 s3_error_name_from_code(s3_error_code_t s3_error_code)
315 if (s3_error_code >= S3_ERROR_END)
316 s3_error_code = S3_ERROR_Unknown;
318 if (s3_error_code == 0)
321 return s3_error_code_names[s3_error_code];
325 /* {{{ s3_curl_supports_ssl */
327 s3_curl_supports_ssl(void)
329 static int supported = -1;
331 if (supported == -1) {
332 #if defined(CURL_VERSION_SSL)
333 curl_version_info_data *info = curl_version_info(CURLVERSION_NOW);
334 if (info->features & CURL_VERSION_SSL)
347 /* {{{ lookup_result */
349 lookup_result(const result_handling_t *result_handling,
351 s3_error_code_t s3_error_code,
354 g_return_val_if_fail(result_handling != NULL, S3_RESULT_FAIL);
356 while (result_handling->response_code
357 || result_handling->s3_error_code
358 || result_handling->curl_code) {
359 if ((result_handling->response_code && result_handling->response_code != response_code)
360 || (result_handling->s3_error_code && result_handling->s3_error_code != s3_error_code)
361 || (result_handling->curl_code && result_handling->curl_code != curl_code)) {
366 return result_handling->result;
369 /* return the result for the terminator, as the default */
370 return result_handling->result;
374 /* {{{ build_resource */
376 build_resource(const char *bucket,
379 char *esc_bucket = NULL, *esc_key = NULL;
380 char *resource = NULL;
383 if (!(esc_bucket = curl_escape(bucket, 0)))
387 if (!(esc_key = curl_escape(key, 0)))
392 resource = g_strdup_printf("/%s/%s", esc_bucket, esc_key);
394 resource = g_strdup_printf("/%s", esc_bucket);
397 resource = g_strdup("/");
400 if (esc_bucket) curl_free(esc_bucket);
401 if (esc_key) curl_free(esc_key);
407 /* {{{ authenticate_request */
408 static struct curl_slist *
409 authenticate_request(S3Handle *hdl,
411 const char *resource)
418 char md_value[EVP_MAX_MD_SIZE+1];
419 char auth_base64[40];
421 struct curl_slist *headers = NULL;
424 /* calculate the date */
426 if (!localtime_r(&t, &tmp)) perror("localtime");
427 if (!strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S %Z", &tmp))
430 /* run HMAC-SHA1 on the canonicalized string */
432 HMAC_Init_ex(&ctx, hdl->secret_key, strlen(hdl->secret_key), EVP_sha1(), NULL);
433 auth_string = g_strconcat(verb, "\n\n\n", date, "\n",
435 AMAZON_SECURITY_HEADER, ":",
436 hdl->user_token, ",",
437 STS_PRODUCT_TOKEN, "\n",
440 HMAC_Update(&ctx, (unsigned char*) auth_string, strlen(auth_string));
442 md_len = EVP_MAX_MD_SIZE;
443 HMAC_Final(&ctx, (unsigned char*)md_value, &md_len);
444 HMAC_CTX_cleanup(&ctx);
445 base64_encode(md_value, md_len, auth_base64, sizeof(auth_base64));
447 /* append the new headers */
449 /* Devpay headers are included in hash. */
450 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", hdl->user_token);
451 headers = curl_slist_append(headers, buf);
454 buf = g_strdup_printf(AMAZON_SECURITY_HEADER ": %s", STS_PRODUCT_TOKEN);
455 headers = curl_slist_append(headers, buf);
459 buf = g_strdup_printf("Authorization: AWS %s:%s",
460 hdl->access_key, auth_base64);
461 headers = curl_slist_append(headers, buf);
464 buf = g_strdup_printf("Date: %s", date);
465 headers = curl_slist_append(headers, buf);
472 /* {{{ interpret_response */
474 regex_error(regex_t *regex, int reg_result)
479 size = regerror(reg_result, regex, NULL, 0);
480 message = g_malloc(size);
481 if (!message) abort(); /* we're really out of luck */
482 regerror(reg_result, regex, message, size);
484 /* this is programmer error (bad regexp), so just log
485 * and abort(). There's no good way to signal a
486 * permanaent error from interpret_response. */
487 g_error(_("Regex error: %s"), message);
488 g_assert_not_reached();
492 interpret_response(S3Handle *hdl,
494 char *curl_error_buffer,
498 long response_code = 0;
499 regmatch_t pmatch[2];
501 char *error_name = NULL, *message = NULL;
502 char *body_copy = NULL;
504 if (!hdl) return FALSE;
506 if (hdl->last_message) g_free(hdl->last_message);
507 hdl->last_message = NULL;
509 /* bail out from a CURL error */
510 if (curl_code != CURLE_OK) {
511 hdl->last_curl_code = curl_code;
512 hdl->last_message = g_strdup_printf("CURL error: %s", curl_error_buffer);
516 /* CURL seems to think things were OK, so get its response code */
517 curl_easy_getinfo(hdl->curl, CURLINFO_RESPONSE_CODE, &response_code);
518 hdl->last_response_code = response_code;
520 /* 2xx and 3xx codes won't have a response body*/
521 if (200 <= response_code && response_code < 400) {
522 hdl->last_s3_error_code = S3_ERROR_None;
526 /* Now look at the body to try to get the actual Amazon error message. Rather
527 * than parse out the XML, just use some regexes. */
529 /* impose a reasonable limit on body size */
530 if (body_len > MAX_ERROR_RESPONSE_LEN) {
531 hdl->last_message = g_strdup("S3 Error: Unknown (response body too large to parse)");
533 } else if (!body || body_len == 0) {
534 hdl->last_message = g_strdup("S3 Error: Unknown (empty response body)");
535 return TRUE; /* perhaps a network error; retry the request */
538 /* use strndup to get a zero-terminated string */
539 body_copy = g_strndup(body, body_len);
540 if (!body_copy) goto cleanup;
542 reg_result = regexec(&error_name_regex, body_copy, 2, pmatch, 0);
543 if (reg_result != 0) {
544 if (reg_result == REG_NOMATCH) {
547 regex_error(&error_name_regex, reg_result);
548 g_assert_not_reached();
551 error_name = find_regex_substring(body_copy, pmatch[1]);
554 reg_result = regexec(&message_regex, body_copy, 2, pmatch, 0);
555 if (reg_result != 0) {
556 if (reg_result == REG_NOMATCH) {
559 regex_error(&message_regex, reg_result);
560 g_assert_not_reached();
563 message = find_regex_substring(body_copy, pmatch[1]);
567 hdl->last_s3_error_code = s3_error_code_from_name(error_name);
571 hdl->last_message = message;
572 message = NULL; /* steal the reference to the string */
576 if (body_copy) g_free(body_copy);
577 if (message) g_free(message);
578 if (error_name) g_free(error_name);
584 /* {{{ perform_request */
585 size_t buffer_readfunction(void *ptr, size_t size,
586 size_t nmemb, void * stream) {
587 CurlBuffer *data = stream;
588 guint bytes_desired = size * nmemb;
590 /* check the number of bytes remaining, just to be safe */
591 if (bytes_desired > data->buffer_len - data->buffer_pos)
592 bytes_desired = data->buffer_len - data->buffer_pos;
594 memcpy((char *)ptr, data->buffer + data->buffer_pos, bytes_desired);
595 data->buffer_pos += bytes_desired;
597 return bytes_desired;
601 buffer_writefunction(void *ptr, size_t size, size_t nmemb, void *stream)
603 CurlBuffer * data = stream;
604 guint new_bytes = size * nmemb;
605 guint bytes_needed = data->buffer_pos + new_bytes;
607 /* error out if the new size is greater than the maximum allowed */
608 if (data->max_buffer_size && bytes_needed > data->max_buffer_size)
611 /* reallocate if necessary. We use exponential sizing to make this
612 * happen less often. */
613 if (bytes_needed > data->buffer_len) {
614 guint new_size = MAX(bytes_needed, data->buffer_len * 2);
615 if (data->max_buffer_size) {
616 new_size = MIN(new_size, data->max_buffer_size);
618 data->buffer = g_realloc(data->buffer, new_size);
619 data->buffer_len = new_size;
621 g_return_val_if_fail(data->buffer, 0); /* returning zero signals an error to libcurl */
623 /* actually copy the data to the buffer */
624 memcpy(data->buffer + data->buffer_pos, ptr, new_bytes);
625 data->buffer_pos += new_bytes;
627 /* signal success to curl */
632 curl_debug_message(CURL *curl G_GNUC_UNUSED,
636 void *unused G_GNUC_UNUSED)
640 char **lines, **line;
647 case CURLINFO_HEADER_IN:
648 lineprefix="Hdr In: ";
651 case CURLINFO_HEADER_OUT:
652 lineprefix="Hdr Out: ";
656 /* ignore data in/out -- nobody wants to see that in the
661 /* split the input into lines */
662 message = g_strndup(s, len);
663 lines = g_strsplit(message, "\n", -1);
666 for (line = lines; *line; line++) {
667 if (**line == '\0') continue; /* skip blank lines */
668 g_debug("%s%s", lineprefix, *line);
676 perform_request(S3Handle *hdl,
677 const char *resource,
680 const void *request_body,
681 guint request_body_size,
682 guint max_response_size,
683 guint preallocate_response_size,
684 const result_handling_t *result_handling)
688 s3_result_t result = S3_RESULT_FAIL; /* assume the worst.. */
689 CURLcode curl_code = CURLE_OK;
690 char curl_error_buffer[CURL_ERROR_SIZE] = "";
691 struct curl_slist *headers = NULL;
692 CurlBuffer readdata = { (void*)request_body, request_body_size, 0, 0 };
693 CurlBuffer writedata = { NULL, 0, 0, max_response_size };
694 gboolean should_retry;
696 gulong backoff = EXPONENTIAL_BACKOFF_START_USEC;
698 g_return_val_if_fail(hdl != NULL && hdl->curl != NULL, S3_RESULT_FAIL);
702 baseurl = s3_curl_supports_ssl()? "https://s3.amazonaws.com":"http://s3.amazonaws.com";
703 url = g_strconcat(baseurl, uri, NULL);
704 if (!url) goto cleanup;
706 if (preallocate_response_size) {
707 writedata.buffer = g_malloc(preallocate_response_size);
708 if (!writedata.buffer) goto cleanup;
709 writedata.buffer_len = preallocate_response_size;
715 curl_slist_free_all(headers);
717 readdata.buffer_pos = 0;
718 writedata.buffer_pos = 0;
719 curl_error_buffer[0] = '\0';
721 /* set up the request */
722 headers = authenticate_request(hdl, verb, resource);
724 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_VERBOSE, hdl->verbose)))
727 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_DEBUGFUNCTION,
728 curl_debug_message)))
730 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_ERRORBUFFER,
733 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_NOPROGRESS, 1)))
735 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_URL, url)))
737 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_HTTPHEADER,
740 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_CUSTOMREQUEST,
743 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEFUNCTION, buffer_writefunction)))
745 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_WRITEDATA, &writedata)))
747 if (max_response_size) {
748 #ifdef CURLOPT_MAXFILESIZE_LARGE
749 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)max_response_size)))
752 # ifdef CURLOPT_MAXFILESIZE
753 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_MAXFILESIZE, (long)max_response_size)))
756 /* no MAXFILESIZE option -- that's OK */
762 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 1)))
764 #ifdef CURLOPT_INFILESIZE_LARGE
765 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)request_body_size)))
768 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_INFILESIZE, (long)request_body_size)))
771 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION, buffer_readfunction)))
773 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA, &readdata)))
776 /* Clear request_body options. */
777 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_UPLOAD, 0)))
779 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READFUNCTION,
782 if ((curl_code = curl_easy_setopt(hdl->curl, CURLOPT_READDATA,
787 /* Perform the request */
788 curl_code = curl_easy_perform(hdl->curl);
791 /* interpret the response into hdl->last* */
792 curl_error: /* (label for short-circuiting the curl_easy_perform call) */
793 should_retry = interpret_response(hdl, curl_code, curl_error_buffer,
794 writedata.buffer, writedata.buffer_pos);
796 /* and, unless we know we need to retry, see what we're to do now */
798 result = lookup_result(result_handling, hdl->last_response_code,
799 hdl->last_s3_error_code, hdl->last_curl_code);
801 /* break out of the while(1) unless we're retrying */
802 if (result != S3_RESULT_RETRY)
806 if (retries >= EXPONENTIAL_BACKOFF_MAX_RETRIES) {
807 /* we're out of retries, so annotate hdl->last_message appropriately and bail
809 char *m = g_strdup_printf("Too many retries; last message was '%s'", hdl->last_message);
810 if (hdl->last_message) g_free(hdl->last_message);
811 hdl->last_message = m;
812 result = S3_RESULT_FAIL;
818 backoff *= EXPONENTIAL_BACKOFF_BASE;
821 if (result != S3_RESULT_OK) {
822 g_debug(_("%s %s failed with %d/%s"), verb, url,
823 hdl->last_response_code,
824 s3_error_name_from_code(hdl->last_s3_error_code));
828 if (url) g_free(url);
829 if (headers) curl_slist_free_all(headers);
831 /* we don't deallocate the response body -- we keep it for later */
832 hdl->last_response_body = writedata.buffer;
833 hdl->last_response_body_size = writedata.buffer_pos;
834 hdl->last_num_retries = retries;
841 * Public function implementations
848 char regmessage[1024];
852 reg_result = regcomp(&error_name_regex, error_name_regex_string, REG_EXTENDED | REG_ICASE);
853 if (reg_result != 0) {
854 size = regerror(reg_result, &error_name_regex, regmessage, sizeof(regmessage));
855 g_error(_("Regex error: %s"), regmessage);
859 reg_result = regcomp(&message_regex, message_regex_string, REG_EXTENDED | REG_ICASE);
860 if (reg_result != 0) {
861 size = regerror(reg_result, &message_regex, regmessage, sizeof(regmessage));
862 g_error(_("Regex error: %s"), regmessage);
872 s3_open(const char *access_key,
873 const char *secret_key
876 const char *user_token
881 hdl = g_new0(S3Handle, 1);
882 if (!hdl) goto error;
884 hdl->verbose = FALSE;
886 hdl->access_key = g_strdup(access_key);
887 if (!hdl->access_key) goto error;
889 hdl->secret_key = g_strdup(secret_key);
890 if (!hdl->secret_key) goto error;
893 hdl->user_token = g_strdup(user_token);
894 if (!hdl->user_token) goto error;
897 hdl->curl = curl_easy_init();
898 if (!hdl->curl) goto error;
910 s3_free(S3Handle *hdl)
915 if (hdl->access_key) g_free(hdl->access_key);
916 if (hdl->secret_key) g_free(hdl->secret_key);
918 if (hdl->user_token) g_free(hdl->user_token);
920 if (hdl->curl) curl_easy_cleanup(hdl->curl);
929 s3_reset(S3Handle *hdl)
932 /* We don't call curl_easy_reset here, because doing that in curl
933 * < 7.16 blanks the default CA certificate path, and there's no way
935 if (hdl->last_message) {
936 g_free(hdl->last_message);
937 hdl->last_message = NULL;
940 hdl->last_response_code = 0;
941 hdl->last_curl_code = 0;
942 hdl->last_s3_error_code = 0;
943 hdl->last_num_retries = 0;
945 if (hdl->last_response_body) {
946 g_free(hdl->last_response_body);
947 hdl->last_response_body = NULL;
950 hdl->last_response_body_size = 0;
957 s3_error(S3Handle *hdl,
958 const char **message,
959 guint *response_code,
960 s3_error_code_t *s3_error_code,
961 const char **s3_error_name,
966 if (message) *message = hdl->last_message;
967 if (response_code) *response_code = hdl->last_response_code;
968 if (s3_error_code) *s3_error_code = hdl->last_s3_error_code;
969 if (s3_error_name) *s3_error_name = s3_error_name_from_code(hdl->last_s3_error_code);
970 if (curl_code) *curl_code = hdl->last_curl_code;
971 if (num_retries) *num_retries = hdl->last_num_retries;
973 /* no hdl? return something coherent, anyway */
974 if (message) *message = "NULL S3Handle";
975 if (response_code) *response_code = 0;
976 if (s3_error_code) *s3_error_code = 0;
977 if (s3_error_name) *s3_error_name = NULL;
978 if (curl_code) *curl_code = 0;
979 if (num_retries) *num_retries = 0;
986 s3_verbose(S3Handle *hdl, gboolean verbose)
988 hdl->verbose = verbose;
994 s3_strerror(S3Handle *hdl)
998 const char *s3_error_name;
1002 char s3_info[256] = "";
1003 char response_info[16] = "";
1004 char curl_info[32] = "";
1005 char retries_info[32] = "";
1007 s3_error(hdl, &message, &response_code, NULL, &s3_error_name, &curl_code, &num_retries);
1010 message = "Unkonwn S3 error";
1012 g_snprintf(s3_info, sizeof(s3_info), " (%s)", s3_error_name);
1014 g_snprintf(response_info, sizeof(response_info), " (HTTP %d)", response_code);
1016 g_snprintf(curl_info, sizeof(curl_info), " (CURLcode %d)", curl_code);
1018 g_snprintf(retries_info, sizeof(retries_info), " (after %d retries)", num_retries);
1020 return g_strdup_printf("%s%s%s%s%s", message, s3_info, curl_info, response_info, retries_info);
1025 /* Perform an upload. When this function returns, KEY and
1026 * BUFFER remain the responsibility of the caller.
1028 * @param self: the s3 device
1029 * @param key: the key to which the upload should be made
1030 * @param buffer: the data to be uploaded
1031 * @param buffer_len: the length of the data to upload
1032 * @returns: false if an error ocurred
1035 s3_upload(S3Handle *hdl,
1041 char *resource = NULL;
1042 s3_result_t result = S3_RESULT_FAIL;
1043 static result_handling_t result_handling[] = {
1044 { 200, 0, 0, S3_RESULT_OK },
1045 RESULT_HANDLING_ALWAYS_RETRY,
1046 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1049 g_return_val_if_fail(hdl != NULL, FALSE);
1051 resource = build_resource(bucket, key);
1053 result = perform_request(hdl, resource, resource, "PUT",
1054 buffer, buffer_len, MAX_ERROR_RESPONSE_LEN, 0,
1059 return result == S3_RESULT_OK;
1063 /* {{{ s3_list_keys */
1065 /* Private structure for our "thunk", which tracks where the user is in the list
1067 struct list_keys_thunk {
1068 GSList *filename_list; /* all pending filenames */
1070 gboolean in_contents; /* look for "key" entities in here */
1071 gboolean in_common_prefixes; /* look for "prefix" entities in here */
1073 gboolean is_truncated;
1082 /* Functions for a SAX parser to parse the XML from Amazon */
1085 list_start_element(GMarkupParseContext *context G_GNUC_UNUSED,
1086 const gchar *element_name,
1087 const gchar **attribute_names G_GNUC_UNUSED,
1088 const gchar **attribute_values G_GNUC_UNUSED,
1090 GError **error G_GNUC_UNUSED)
1092 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1094 thunk->want_text = 0;
1095 if (strcasecmp(element_name, "contents") == 0) {
1096 thunk->in_contents = 1;
1097 } else if (strcasecmp(element_name, "commonprefixes") == 0) {
1098 thunk->in_common_prefixes = 1;
1099 } else if (strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1100 thunk->want_text = 1;
1101 } else if (strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1102 thunk->want_text = 1;
1103 } else if (strcasecmp(element_name, "istruncated")) {
1104 thunk->want_text = 1;
1105 } else if (strcasecmp(element_name, "nextmarker")) {
1106 thunk->want_text = 1;
1111 list_end_element(GMarkupParseContext *context G_GNUC_UNUSED,
1112 const gchar *element_name,
1114 GError **error G_GNUC_UNUSED)
1116 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1118 if (strcasecmp(element_name, "contents") == 0) {
1119 thunk->in_contents = 0;
1120 } else if (strcasecmp(element_name, "commonprefixes") == 0) {
1121 thunk->in_common_prefixes = 0;
1122 } else if (strcasecmp(element_name, "key") == 0 && thunk->in_contents) {
1123 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1125 } else if (strcasecmp(element_name, "prefix") == 0 && thunk->in_common_prefixes) {
1126 thunk->filename_list = g_slist_prepend(thunk->filename_list, thunk->text);
1128 } else if (strcasecmp(element_name, "istruncated") == 0) {
1129 if (thunk->text && strncasecmp(thunk->text, "false", 5) != 0)
1130 thunk->is_truncated = TRUE;
1131 } else if (strcasecmp(element_name, "nextmarker") == 0) {
1132 if (thunk->next_marker) g_free(thunk->next_marker);
1133 thunk->next_marker = thunk->text;
1139 list_text(GMarkupParseContext *context G_GNUC_UNUSED,
1143 GError **error G_GNUC_UNUSED)
1145 struct list_keys_thunk *thunk = (struct list_keys_thunk *)user_data;
1147 if (thunk->want_text) {
1148 if (thunk->text) g_free(thunk->text);
1149 thunk->text = g_strndup(text, text_len);
1153 /* Helper function for list_fetch */
1155 list_build_url_component(char **rv,
1160 char *esc_value = NULL;
1161 char *new_rv = NULL;
1163 esc_value = curl_escape(value, 0);
1164 if (!esc_value) goto cleanup;
1166 new_rv = g_strconcat(*rv, delim, key, "=", esc_value, NULL);
1167 if (!new_rv) goto cleanup;
1171 curl_free(esc_value);
1176 if (new_rv) g_free(new_rv);
1177 if (esc_value) curl_free(esc_value);
1182 /* Perform a fetch from S3; several fetches may be involved in a
1183 * single listing operation */
1185 list_fetch(S3Handle *hdl,
1186 const char *resource,
1188 const char *delimiter,
1190 const char *max_keys)
1192 char *urldelim = "?";
1193 char *uri = g_strdup(resource);
1194 s3_result_t result = S3_RESULT_FAIL;
1195 static result_handling_t result_handling[] = {
1196 { 200, 0, 0, S3_RESULT_OK },
1197 RESULT_HANDLING_ALWAYS_RETRY,
1198 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1203 if (!list_build_url_component(&uri, urldelim, "prefix", prefix)) goto cleanup;
1207 if (!list_build_url_component(&uri, urldelim, "delimiter", delimiter)) goto cleanup;
1211 if (!list_build_url_component(&uri, urldelim, "marker", marker)) goto cleanup;
1215 if (!list_build_url_component(&uri, urldelim, "max-keys", max_keys)) goto cleanup;
1219 /* and perform the request on that URI */
1220 result = perform_request(hdl, resource, uri, "GET", NULL,
1221 0, MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1224 if (uri) g_free(uri);
1229 s3_list_keys(S3Handle *hdl,
1232 const char *delimiter,
1235 char *resource = NULL;
1236 struct list_keys_thunk thunk;
1237 GMarkupParseContext *ctxt = NULL;
1238 static GMarkupParser parser = { list_start_element, list_end_element, list_text, NULL, NULL };
1240 s3_result_t result = S3_RESULT_FAIL;
1244 thunk.filename_list = NULL;
1246 thunk.next_marker = NULL;
1248 resource = build_resource(bucket, NULL);
1249 if (!resource) goto cleanup;
1251 /* Loop until S3 has given us the entire picture */
1253 /* get some data from S3 */
1254 result = list_fetch(hdl, resource, prefix, delimiter, thunk.next_marker, NULL);
1255 if (result != S3_RESULT_OK) goto cleanup;
1257 /* run the parser over it */
1258 thunk.in_contents = FALSE;
1259 thunk.in_common_prefixes = FALSE;
1260 thunk.is_truncated = FALSE;
1261 thunk.want_text = FALSE;
1263 ctxt = g_markup_parse_context_new(&parser, 0, (gpointer)&thunk, NULL);
1265 if (!g_markup_parse_context_parse(ctxt, hdl->last_response_body,
1266 hdl->last_response_body_size, &err)) {
1267 if (hdl->last_message) g_free(hdl->last_message);
1268 hdl->last_message = g_strdup(err->message);
1269 result = S3_RESULT_FAIL;
1273 if (!g_markup_parse_context_end_parse(ctxt, &err)) {
1274 if (hdl->last_message) g_free(hdl->last_message);
1275 hdl->last_message = g_strdup(err->message);
1276 result = S3_RESULT_FAIL;
1280 g_markup_parse_context_free(ctxt);
1282 } while (thunk.next_marker);
1285 if (err) g_error_free(err);
1286 if (thunk.text) g_free(thunk.text);
1287 if (thunk.next_marker) g_free(thunk.next_marker);
1288 if (resource) g_free(resource);
1289 if (ctxt) g_markup_parse_context_free(ctxt);
1291 if (result != S3_RESULT_OK) {
1292 g_slist_free(thunk.filename_list);
1295 *list = thunk.filename_list;
1303 s3_read(S3Handle *hdl,
1310 char *resource = NULL;
1311 s3_result_t result = S3_RESULT_FAIL;
1312 static result_handling_t result_handling[] = {
1313 { 200, 0, 0, S3_RESULT_OK },
1314 RESULT_HANDLING_ALWAYS_RETRY,
1315 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1318 g_return_val_if_fail(hdl != NULL, FALSE);
1319 g_assert(buf_ptr != NULL);
1320 g_assert(buf_size != NULL);
1325 resource = build_resource(bucket, key);
1327 result = perform_request(hdl, resource, resource,
1328 "GET", NULL, 0, max_size, 0, result_handling);
1331 /* copy the pointer to the result parameters and remove
1332 * our reference to it */
1333 if (result == S3_RESULT_OK) {
1334 *buf_ptr = hdl->last_response_body;
1335 *buf_size = hdl->last_response_body_size;
1337 hdl->last_response_body = NULL;
1338 hdl->last_response_body_size = 0;
1342 return result == S3_RESULT_OK;
1348 s3_delete(S3Handle *hdl,
1352 char *resource = NULL;
1353 s3_result_t result = S3_RESULT_FAIL;
1354 static result_handling_t result_handling[] = {
1355 { 204, 0, 0, S3_RESULT_OK },
1356 RESULT_HANDLING_ALWAYS_RETRY,
1357 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1360 g_return_val_if_fail(hdl != NULL, FALSE);
1362 resource = build_resource(bucket, key);
1364 result = perform_request(hdl, resource, resource, "DELETE", NULL, 0,
1365 MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1369 return result == S3_RESULT_OK;
1373 /* {{{ s3_make_bucket */
1375 s3_make_bucket(S3Handle *hdl,
1378 char *resource = NULL;
1379 s3_result_t result = result = S3_RESULT_FAIL;
1380 static result_handling_t result_handling[] = {
1381 { 200, 0, 0, S3_RESULT_OK },
1382 RESULT_HANDLING_ALWAYS_RETRY,
1383 { 0, 0, 0, /* default: */ S3_RESULT_FAIL }
1386 g_return_val_if_fail(hdl != NULL, FALSE);
1388 resource = build_resource(bucket, NULL);
1390 result = perform_request(hdl, resource, resource, "PUT", NULL, 0,
1391 MAX_ERROR_RESPONSE_LEN, 0, result_handling);
1395 return result == S3_RESULT_OK;