2 * Copyright (c) 2008-2012 Zmanda, Inc. All Rights Reserved.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 as published
6 * by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
10 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17 * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
18 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
24 #include <curl/curl.h>
37 /* An opaque handle. S3Handles should only be accessed from a single
38 * thread at any given time, although it is fine to use different handles
39 * in different threads simultaneously. */
40 typedef struct S3Handle S3Handle;
42 /* Callback function to read data to upload
44 * @note this is the same as CURLOPT_READFUNCTION
46 * @param data: The pointer to write data to
47 * @param size: The size of each "element" of the data buffer in bytes
48 * @param nmemb: The number of elements in the data buffer.
49 * So, the buffer's size is size*nmemb bytes.
50 * @param stream: The read_data (an opaque pointer)
52 * @return The number of bytes written to the buffer,
53 * CURL_READFUNC_PAUSE to pause, or CURL_READFUNC_ABORT to abort.
54 * Return 0 only if there's no more data to be uploaded.
56 typedef size_t (*s3_read_func)(void *data, size_t size, size_t nmemb, void *stream);
58 /* This function is called to get size of the upload data
60 * @param data: The write_data (opaque pointer)
62 * @return The number of bytes of data, negative for error
64 typedef size_t (*s3_size_func)(void *data);
66 /* This function is called to get MD5 hash of the upload data
68 * @param data: The write_data (opaque pointer)
70 * @return The MD5 hash, NULL on error
72 typedef GByteArray* (*s3_md5_func)(void *data);
74 /* This function is called to reset an upload or download data stream
77 * @param data: The read_data or write_data (opaque pointer)
79 * @return The number of bytes of data, negative for error
81 typedef void (*s3_reset_func)(void *data);
83 /* Callback function to write data that's been downloaded
85 * @note this is the same as CURLOPT_WRITEFUNCTION
87 * @param data: The pointer to read data from
88 * @param size: The size of each "element" of the data buffer in bytes
89 * @param nmemb: The number of elements in the data buffer.
90 * So, the buffer's size is size*nmemb bytes.
91 * @param stream: the write_data (an opaque pointer)
93 * @return The number of bytes written to the buffer or
94 * CURL_WRITEFUNC_PAUSE to pause.
95 * If it's the number of bytes written, it should match the buffer size
97 typedef size_t (*s3_write_func)(void *data, size_t size, size_t nmemb, void *stream);
100 * Callback function to track progress
102 * @note this is the same as CURLOPT_PROGRESSFUNCTION
104 * @param data: The progress_data
105 * @param dltotal: The total number of bytes to downloaded
106 * @param dlnow: The current number of bytes downloaded
107 * @param ultotal: The total number of bytes to downloaded
108 * @param ulnow: The current number of bytes downloaded
110 * @return 0 to continue, non-zero to abort.
112 typedef curl_progress_callback s3_progress_func;
118 /* These are assumed to be already URL-escaped. */
119 # define STS_BASE_URL "https://ls.amazonaws.com/"
120 # define STS_PRODUCT_TOKEN "{ProductToken}AAAGQXBwVGtu4geoGybuwuk8VEEPzJ9ZANpu0yzbf9g4Gs5Iarzff9B7qaDBEEaWcAzWpcN7zmdMO765jOtEFc4DWTRNkpPSzUnTdkHbdYUamath73OreaZtB86jy/JF0gsHZfhxeKc/3aLr8HNT//DsX3r272zYHLDPWWUbFguOwqNjllnt6BshYREx59l8RrWABLSa37dyJeN+faGvz3uQxiDakZRn3LfInOE6d9+fTFl50LPoP08LCqI/SJfpouzWix7D/cep3Jq8yYNyM1rgAOTF7/wh7r8OuPDLJ/xZUDLfykePIAM="
122 /* This preprocessor magic will enumerate constants named S3_ERROR_XxxYyy for
123 * each of the errors in parentheses.
125 * see http://docs.amazonwebservices.com/AmazonS3/latest/API/ErrorResponses.html
126 * for Amazon's breakdown of error responses.
128 #define S3_ERROR_LIST \
130 S3_ERROR(AccountProblem), \
131 S3_ERROR(AllAccessDisabled), \
132 S3_ERROR(AmbiguousGrantByEmailAddress), \
133 S3_ERROR(OperationAborted), \
134 S3_ERROR(BadDigest), \
135 S3_ERROR(BucketAlreadyExists), \
136 S3_ERROR(BucketAlreadyOwnedByYou), \
137 S3_ERROR(BucketNotEmpty), \
138 S3_ERROR(CredentialsNotSupported), \
139 S3_ERROR(EntityTooLarge), \
140 S3_ERROR(IncompleteBody), \
141 S3_ERROR(InternalError), \
142 S3_ERROR(InvalidAccessKeyId), \
143 S3_ERROR(InvalidArgument), \
144 S3_ERROR(InvalidBucketName), \
145 S3_ERROR(InvalidDigest), \
146 S3_ERROR(InvalidRange), \
147 S3_ERROR(InvalidSecurity), \
148 S3_ERROR(InvalidSOAPRequest), \
149 S3_ERROR(InvalidStorageClass), \
150 S3_ERROR(InvalidTargetBucketForLogging), \
151 S3_ERROR(KeyTooLong), \
152 S3_ERROR(InvalidURI), \
153 S3_ERROR(MalformedACLError), \
154 S3_ERROR(MaxMessageLengthExceeded), \
155 S3_ERROR(MetadataTooLarge), \
156 S3_ERROR(MethodNotAllowed), \
157 S3_ERROR(MissingAttachment), \
158 S3_ERROR(MissingContentLength), \
159 S3_ERROR(MissingSecurityElement), \
160 S3_ERROR(MissingSecurityHeader), \
161 S3_ERROR(NoLoggingStatusForKey), \
162 S3_ERROR(NoSuchBucket), \
163 S3_ERROR(NoSuchEntity), \
164 S3_ERROR(NoSuchKey), \
165 S3_ERROR(NotImplemented), \
166 S3_ERROR(NotSignedUp), \
167 S3_ERROR(PreconditionFailed), \
168 S3_ERROR(RequestTimeout), \
169 S3_ERROR(RequestTimeTooSkewed), \
170 S3_ERROR(RequestTorrentOfBucketError), \
171 S3_ERROR(SignatureDoesNotMatch), \
172 S3_ERROR(TooManyBuckets), \
173 S3_ERROR(UnexpectedContent), \
174 S3_ERROR(UnresolvableGrantByEmailAddress), \
176 S3_ERROR(Unauthorized), \
178 S3_ERROR(Accepted), \
179 S3_ERROR(Forbidden), \
180 S3_ERROR(Conflict), \
181 S3_ERROR(AuthenticationRequired), \
185 #define S3_ERROR(NAME) S3_ERROR_ ## NAME
194 /* Does this install of curl support SSL?
199 s3_curl_supports_ssl(void);
201 /* Checks if the version of libcurl being used supports and checks
202 * wildcard certificates correctly (used for the subdomains required
203 * by location constraints).
205 * @returns: true if the version of libcurl is new enough
208 s3_curl_location_compat(void);
210 /* Checks if a bucket name is compatible with setting a location
213 * @note This doesn't guarantee that bucket name is entirely valid,
214 * just that using it as one (or more) subdomain(s) of s3.amazonaws.com
215 * won't fail; that would prevent the reporting of useful messages from
218 * @param bucket: the bucket name
219 * @returns: true if the bucket name is compatible
222 s3_bucket_location_compat(const char *bucket);
224 /* Initialize S3 operation
226 * If an error occurs in this function, diagnostic information is
229 * @returns: false if an error occurred
234 /* Set up an S3Handle.
236 * The concept of a bucket is defined by the Amazon S3 API.
237 * See: "Components of Amazon S3" - API Version 2006-03-01 pg. 8
239 * @param access_key: the secret key for Amazon Web Services
240 * @param secret_key: the secret key for Amazon Web Services
241 * @param user_token: the user token for Amazon DevPay
242 * @param bucket_location: the location constraint for buckets
243 * @param storage_class: the storage class for new objects
244 * @param ca_info: the path to pass to libcurl as the certificate authority.
245 * see curl_easy_setopt() CURLOPT_CAINFO for more
246 * @returns: the new S3Handle
249 s3_open(const char * access_key, const char *secret_key,
250 const char *swift_account_id, const char *swift_access_key,
252 const char *service_path, gboolean use_subdomain,
253 const char * user_token,
254 const char * bucket_location, const char * storage_class,
255 const char * ca_info, const char * server_side_encryption,
258 const char *username,
259 const char *password,
260 const char *tenant_id,
261 const char *tenant_name,
262 const char *client_id,
263 const char *client_secret,
264 const char *refresh_token,
265 const gboolean reuse_connection);
267 /* latest step of setting up the S3Handle.
269 * Must be done after all properties are set.
271 * @param hdl: the S3Handle to set up.
272 * @returns: false if an error occured
275 s3_open2(S3Handle *hdl);
277 /* Deallocate an S3Handle
279 * @param hdl: the S3Handle object
282 s3_free(S3Handle *hdl);
284 /* Reset the information about the last request, including
285 * freeing any allocated memory. The S3Handle itself is not
286 * freed and may be used again. This function is called
287 * automatically as needed, and should be called to free memory
288 * when the handle will not be used for some time.
290 * @param hdl: the S3Handle object
293 s3_reset(S3Handle *hdl);
295 /* Get the error information for the last operation
297 * All results are returned via result parameters. If any parameter is
298 * NULL, that result will not be returned. Caller is not responsible for
299 * freeing any returned strings, although the results are only valid until
300 * the next call to an S3 function with this handle.
302 * @param hdl: the S3Handle object
303 * @param message: (result) the error message, or NULL if none exists
304 * @param response_code: (result) the HTTP response code (or 0 if none exists)
305 * @param s3_error_code: (result) the S3 error code (see constants, above)
306 * @param s3_error_name: (result) the S3 error name (e.g., "RequestTimeout"),
307 * or NULL if none exists
308 * @param curl_code: (result) the curl error code (or 0 if none exists)
309 * @param num_retries: (result) number of retries
312 s3_error(S3Handle *hdl,
313 const char **message,
314 guint *response_code,
315 s3_error_code_t *s3_error_code,
316 const char **s3_error_name,
320 /* Control verbose output of HTTP transactions, etc.
322 * @param hdl: the S3Handle object
323 * @param verbose: if true, send HTTP transactions, etc. to debug output
326 s3_verbose(S3Handle *hdl,
329 /* Control the use of SSL with HTTP transactions.
331 * @param hdl: the S3Handle object
332 * @param use_ssl: if true, use SSL (if curl supports it)
333 * @returns: true if the setting is valid
336 s3_use_ssl(S3Handle *hdl, gboolean use_ssl);
338 /* Control the throttling of S3 uploads. Only supported with curl >= 7.15.5.
340 * @param hdl: the S3Handle object
341 * @param max_send_speed: max speed (bytes/sec) at which to send
342 * @returns: true if the setting is valid
345 s3_set_max_send_speed(S3Handle *hdl, guint64 max_send_speed);
347 /* Control the throttling of S3 downloads. Only supported with curl >= 7.15.5.
349 * @param hdl: the S3Handle object
350 * @param max_recv_speed: max speed (bytes/sec) at which to receive
351 * @returns: true if the setting is valid
354 s3_set_max_recv_speed(S3Handle *hdl, guint64 max_recv_speed);
356 /* Get the error information from the last operation on this handle,
357 * formatted as a string.
359 * Caller is responsible for freeing the resulting string.
361 * @param hdl: the S3Handle object
362 * @returns: string, or NULL if no error occurred
365 s3_strerror(S3Handle *hdl);
367 /* Perform an upload.
369 * When this function returns, KEY and BUFFER remain the
370 * responsibility of the caller.
372 * @param hdl: the S3Handle object
373 * @param bucket: the bucket to which the upload should be made
374 * @param key: the key to which the upload should be made
375 * @param read_func: the callback for reading data
376 * @param reset_func: the callback for to reset reading data
377 * @param size_func: the callback to get the number of bytes to upload
378 * @param md5_func: the callback to get the MD5 hash of the data to upload
379 * @param read_data: pointer to pass to the above functions
380 * @param progress_func: the callback for progress information
381 * @param progress_data: pointer to pass to C{progress_func}
383 * @returns: false if an error ocurred
386 s3_upload(S3Handle *hdl,
389 s3_read_func read_func,
390 s3_reset_func reset_func,
391 s3_size_func size_func,
392 s3_md5_func md5_func,
394 s3_progress_func progress_func,
395 gpointer progress_data);
397 /* List all of the files matching the pseudo-glob C{PREFIX*DELIMITER*},
398 * returning only that portion which matches C{PREFIX*DELIMITER}. S3 supports
399 * this particular semantics, making it quite efficient. The returned list
400 * should be freed by the caller.
402 * @param hdl: the S3Handle object
403 * @param bucket: the bucket to list
404 * @param prefix: the prefix
405 * @param delimiter: delimiter (any length string)
406 * @param list: (output) the list of files
407 * @param total_size: (output) sum of size of files
408 * @returns: FALSE if an error occurs
411 s3_list_keys(S3Handle *hdl,
414 const char *delimiter,
416 guint64 *total_size);
418 /* Read an entire file, passing the contents to write_func buffer
421 * @param hdl: the S3Handle object
422 * @param bucket: the bucket to read from
423 * @param key: the key to read from
424 * @param write_func: the callback for writing data
425 * @param reset_func: the callback for to reset writing data
426 * @param write_data: pointer to pass to C{write_func}
427 * @param progress_func: the callback for progress information
428 * @param progress_data: pointer to pass to C{progress_func}
429 * @returns: FALSE if an error occurs
432 s3_read(S3Handle *hdl,
435 s3_write_func write_func,
436 s3_reset_func reset_func,
438 s3_progress_func progress_func,
439 gpointer progress_data);
443 * @param hdl: the S3Handle object
444 * @param bucket: the bucket to delete from
445 * @param key: the key to delete
446 * @returns: FALSE if an error occurs; a non-existent file is I{not} considered an error.
449 s3_delete(S3Handle *hdl,
453 /* Delete multiple file.
455 * @param hdl: the S3Handle object
456 * @param bucket: the bucket to delete from
457 * @param key: the key array to delete
458 * @returns: 0 on sucess, 1 if multi_delete is not supported, 2 if an error
459 * occurs; a non-existent file is I{not} considered an error.
462 s3_multi_delete(S3Handle *hdl,
468 * @param hdl: the S3Handle object
469 * @param bucket: the bucket to create
470 * @returns: FALSE if an error occurs
473 s3_make_bucket(S3Handle *hdl,
475 const char *project_id);
477 /* Check if a bucket exists.
479 * @param hdl: the S3Handle object
480 * @param bucket: the bucket to create
481 * @returns: FALSE if an error occur
484 s3_is_bucket_exists(S3Handle *hdl,
486 const char *project_id);
490 * @note A bucket can not be deleted if it still contains keys
492 * @param hdl: the S3Handle object
493 * @param bucket: the bucket to delete
494 * @returns: FALSE if an error occurs
497 s3_delete_bucket(S3Handle *hdl,
500 /* Attempt a RefreshAWSSecurityToken on a token; if it succeeds, the old
501 * token will be freed and replaced by the new. If it fails, the old
502 * token is left unchanged and FALSE is returned. */
503 gboolean sts_refresh_token(char ** token, const char * directory);
505 /* These functions are for if you want to use curl on your own. You get more
506 * control, but it's a lot of work that way: */
511 guint max_buffer_size;
514 #define S3_BUFFER_READ_FUNCS s3_buffer_read_func, s3_buffer_reset_func, s3_buffer_size_func, s3_buffer_md5_func
516 #define S3_BUFFER_WRITE_FUNCS s3_buffer_write_func, s3_buffer_reset_func
518 /* a CURLOPT_READFUNCTION to read data from a buffer. */
520 s3_buffer_read_func(void *ptr, size_t size, size_t nmemb, void * stream);
523 s3_buffer_size_func(void *stream);
526 s3_buffer_md5_func(void *stream);
529 s3_buffer_reset_func(void *stream);
531 #define S3_EMPTY_READ_FUNCS s3_empty_read_func, NULL, s3_empty_size_func, s3_empty_md5_func
533 /* a CURLOPT_WRITEFUNCTION to write data to a buffer. */
535 s3_buffer_write_func(void *ptr, size_t size, size_t nmemb, void *stream);
537 /* a CURLOPT_READFUNCTION that writes nothing. */
539 s3_empty_read_func(void *ptr, size_t size, size_t nmemb, void * stream);
542 s3_empty_size_func(void *stream);
545 s3_empty_md5_func(void *stream);
547 #define S3_COUNTER_WRITE_FUNCS s3_counter_write_func, s3_counter_reset_func
549 /* a CURLOPT_WRITEFUNCTION to write data that just counts data.
550 * s3_write_data should be NULL or a pointer to an gint64.
553 s3_counter_write_func(void *ptr, size_t size, size_t nmemb, void *stream);
556 s3_counter_reset_func(void *stream);
559 /* a CURLOPT_READFUNCTION to read data from a file. */
561 s3_file_read_func(void *ptr, size_t size, size_t nmemb, void * stream);
564 s3_file_size_func(void *stream);
567 s3_file_md5_func(void *stream);
570 s3_file_reset_func(void *stream);
572 /* a CURLOPT_WRITEFUNCTION to write data to a file. */
574 s3_file_write_func(void *ptr, size_t size, size_t nmemb, void *stream);
577 /* Adds a null termination to a buffer. */
578 void terminate_buffer(CurlBuffer *);