2 * Amanda, The Advanced Maryland Automatic Network Disk Archiver
3 * Copyright (c) 2008-2012 Zmanda, Inc. All Rights Reserved.
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 * Contact information: Zmanda Inc., 465 S. Mathilda Ave., Suite 300
20 * Sunnyvale, CA 94085, USA, or: http://www.zmanda.com
23 /* Base classes for transfer elements.
26 #ifndef XFER_ELEMENT_H
27 #define XFER_ELEMENT_H
30 #include <glib-object.h>
33 #include "directtcp.h"
36 /* sources have no input mechanisms and destinations have no output
40 /* downstream element will read() from elt->upstream's output_fd; EOF
41 * is indicated by the usual OS mechanism resulting in a zero-length
42 * read, in response to which the downstream element must close
46 /* upstream element will write() to elt->downstream's input_fd. EOF
47 * is indicated by closing the file descriptor. */
50 /* downstream element will call elt->upstream->pull_buffer() to
51 * pull a buffer. EOF is indicated by returning a NULL buffer */
52 XFER_MECH_PULL_BUFFER,
54 /* upstream element will call elt->downstream->push_buffer(buf) to push
55 * a buffer. EOF is indicated by passing a NULL buffer. */
56 XFER_MECH_PUSH_BUFFER,
58 /* DirectTCP: downstream sends an array of IP:PORT addresses to which a TCP
59 * connection should be made, then upstream connects to one of the addreses
60 * and sends the data over that connection */
61 XFER_MECH_DIRECTTCP_LISTEN,
63 /* DirectTCP: downstream gets IP:PORT addresses from upstream to which a
64 * TCP connection should be made, then connects to one of the addreses and
65 * receives the data over that connection */
66 XFER_MECH_DIRECTTCP_CONNECT,
68 /* (sentinel value) */
73 * Description of a pair (input, output) of xfer mechanisms that an
74 * element can support, along with the associated costs. An array of these
75 * pairs is stored in the class-level variable 'mech_pairs', describing
76 * all of the mechanisms that an element supports.
78 * Use the XFER_NROPS() and XFER_NTHREADS() macros below in declarations in
79 * order to make declarations more understandable.
82 #define XFER_NROPS(x) (x)
83 #define XFER_NTHREADS(x) (x)
87 xfer_mech output_mech;
88 guint8 ops_per_byte; /* number of byte copies or other operations */
89 guint8 nthreads; /* number of additional threads created */
90 } xfer_element_mech_pair_t;
92 /***********************
95 * The virtual base class for all transfer elements
98 GType xfer_element_get_type(void);
99 #define XFER_ELEMENT_TYPE (xfer_element_get_type())
100 #define XFER_ELEMENT(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), xfer_element_get_type(), XferElement)
101 #define XFER_ELEMENT_CONST(obj) G_TYPE_CHECK_INSTANCE_CAST((obj), xfer_element_get_type(), XferElement const)
102 #define XFER_ELEMENT_CLASS(klass) G_TYPE_CHECK_CLASS_CAST((klass), xfer_element_get_type(), XferElementClass)
103 #define IS_XFER_ELEMENT(obj) G_TYPE_CHECK_INSTANCE_TYPE((obj), xfer_element_get_type ())
104 #define XFER_ELEMENT_GET_CLASS(obj) G_TYPE_INSTANCE_GET_CLASS((obj), xfer_element_get_type(), XferElementClass)
107 * Main object structure
110 typedef struct XferElement {
113 /* The transfer to which this element is attached */
114 Xfer *xfer; /* set by xfer_new */
116 /* assigned input and output mechanisms */
117 xfer_mech input_mech;
118 xfer_mech output_mech;
120 /* neighboring xfer elements */
121 struct XferElement *upstream;
122 struct XferElement *downstream;
124 /* Information regarding cancellation. Cancelled and expect_eof are set by
125 * the default cancel() method. Can_generate_eof should be set during
126 * initialization, and is returned by the default cancel implementation */
129 gboolean can_generate_eof;
131 /* file descriptors for XFER_MECH_READFD and XFER_MECH_WRITEFD. These
132 * should be set during setup(), and can be accessed by neighboring
133 * elements during start(). These values are shared among multiple
134 * elements, and thus must be accessed with xfer_element_swap_input_fd and
135 * xfer_element_swap_output_fd. Any file descriptors remaining here at
136 * finalize time will be closed. */
140 /* array of IP:PORT pairs that can be used to connect to this element,
141 * terminated by a 0.0.0.0:0. The first is set by elements with an input
142 * mech of XFER_MECH_DIRECTTCP_LISTEN and accessed by their upstream
143 * neighbor; the second is set by elements with an output mech of
144 * XFER_MECH_DIRECTTCP_CONNECT and accessed by their downstream neighbor.
147 DirectTCPAddr *input_listen_addrs;
148 DirectTCPAddr *output_listen_addrs;
150 /* cache for repr() */
153 /* maximum size to transfer */
162 GObjectClass __parent__;
164 /* Get a string representation of this element. The returned string will be freed
165 * when the element is finalized, and is static until that time. This method is
166 * implemented by XferElement, but can be overridden by classes that can provide
167 * additional useful information about themselves. Overriding methods can use
168 * the 'repr' instance variable as a cache -- it will be freed on finalize().
170 * @param elt: the XferElement
171 * @return: statically allocated string
173 char *(*repr)(XferElement *elt);
175 /* Set up this element. This function is called for all elements in a
176 * transfer before start() is called for any elements. For mechanisms
177 * where this element supplies a file descriptor, it should set its
178 * input_fd and/or output_fd appropriately; neighboring elements will use
179 * that value in start(). Elements which supply IP:PORT pairs should set
180 * their input_addrs, for neighboring elements to use in start().
182 * elt->input_mech and elt->output_mech are already set when this function
183 * is called, but upstream and downstream are not.
185 * If the setup operation fails, the method should send an XMSG_ERROR and
186 * call XMSG_CANCEL, and return False. In this situation, the start method
187 * will not be called. The Xfer will appear to the user to start and
190 * Note that this method may never be called if other elements' setup methods
193 * @param elt: the XferElement
194 * @return: false on failure, true on success
196 gboolean (*setup)(XferElement *elt);
198 /* set the size of data to transfer, to skip NUL padding bytes
199 * @param elt: the XferElement
200 * @param size: the size of data to transfer
203 gboolean (*set_size)(XferElement *elt, gint64 size);
205 /* Start transferring data. The element downstream of this one will
206 * already be started, while the upstream element will not, so data will
207 * not begin flowing immediately. It is safe to access attributes of
208 * neighboring elements during this call.
210 * This method will *not* be called if all elements do not set up
213 * @param elt: the XferElement
214 * @return: TRUE if this element will send XMSG_DONE
216 gboolean (*start)(XferElement *elt);
218 /* Stop transferring data. The upstream element's cancel method will
219 * already have been called, but due to buffering and synchronization
220 * issues, data may yet arrive. The element may discard any such data, but
221 * must not fail. This method is only called for abnormal terminations;
222 * elements should normally stop processing on receiving an EOF indication
225 * If expect_eof is TRUE, then this element should expect an EOF from its
226 * upstream element, and should drain any remaining data until that EOF
227 * arrives and generate an EOF to the downstream element. The utility
228 * functions xfer_element_drain_fd and xfer_element_drain_buffers may be
229 * useful for this purpose. This draining is important in order to avoid
230 * hung threads or unexpected SIGPIPEs.
232 * If expect_eof is FALSE, then the upstream elements are unable to
233 * generate an early EOF, so this element should *not* attempt to drain any
234 * remaining data. As an example, an FdSource is not active and thus
235 * cannot generate an EOF on request.
237 * If this element can generate an EOF, it should return TRUE, otherwise
240 * This method may be called before start or setup if an error is
241 * encountered during setup.
243 * The default implementation sets self->expect_eof and self->cancelled
244 * appropriately and returns self->can_generate_eof.
246 * This method is always called from the main thread. It must not block.
248 * @param elt: the XferElement
249 * @param expect_eof: element should expect an EOF
250 * @returns: true if this element can return EOF
252 gboolean (*cancel)(XferElement *elt, gboolean expect_eof);
254 /* Get a buffer full of data from this element. This function is called by
255 * the downstream element under XFER_MECH_PULL_CALL. It can block indefinitely,
256 * and must only return NULL on EOF. Responsibility to free the buffer transfers
259 * @param elt: the XferElement
260 * @param size (output): size of resulting buffer
261 * @returns: buffer pointer
263 gpointer (*pull_buffer)(XferElement *elt, size_t *size);
265 /* A buffer full of data is being sent to this element for processing; this
266 * function is called by the upstream element under XFER_MECH_PUSH_CALL.
267 * It can block indefinitely if the data cannot be processed immediately.
268 * An EOF condition is signaled by call with a NULL buffer. Responsibility to
269 * free the buffer transfers to the callee.
271 * @param elt: the XferElement
273 * @param size: size of buffer
275 void (*push_buffer)(XferElement *elt, gpointer buf, size_t size);
277 /* Returns the mech_pairs that this element supports. The default
278 * implementation just returns the class attribute 'mech_pairs', but
279 * subclasses can dynamically select the available mechanisms by overriding
280 * this method. Note that the method is called before the setup() method.
282 * @param elt: the XferElement
283 * @returns: array of mech pairs, terminated by <NONE,NONE>
285 xfer_element_mech_pair_t *(*get_mech_pairs)(XferElement *elt);
287 /* class variables */
289 /* This is used by the perl bindings -- it is a class variable giving the
290 * appropriate perl class to wrap this XferElement. It should be set by
291 * each class's class_init.
293 const char *perl_class;
295 /* Statically allocated array of input/output mechanisms supported by this
296 * class (terminated by <XFER_MECH_NONE,XFER_MECH_NONE>). The default
297 * get_mech_pairs method returns this. */
298 xfer_element_mech_pair_t *mech_pairs;
305 void xfer_element_unref(XferElement *elt);
306 gboolean xfer_element_link_to(XferElement *elt, XferElement *successor);
307 char *xfer_element_repr(XferElement *elt);
308 gboolean xfer_element_setup(XferElement *elt);
309 gboolean xfer_element_set_size(XferElement *elt, gint64 size);
310 gboolean xfer_element_start(XferElement *elt);
311 void xfer_element_push_buffer(XferElement *elt, gpointer buf, size_t size);
312 gpointer xfer_element_pull_buffer(XferElement *elt, size_t *size);
313 gboolean xfer_element_cancel(XferElement *elt, gboolean expect_eof);
314 xfer_element_mech_pair_t *xfer_element_get_mech_pairs(XferElement *elt);
319 * These are utilities for subclasses
322 /* Drain UPSTREAM by pulling buffers until EOF
324 * @param upstream: the element to drain
326 void xfer_element_drain_buffers(XferElement *upstream);
328 /* Drain UPSTREAM by reading until EOF. This does not close
329 * the file descriptor.
331 * @param fd: the file descriptor to drain
333 void xfer_element_drain_fd(int fd);
335 /* Atomically swap a value into elt->_input_fd and _output_fd, respectively.
336 * Always use these methods to access the field.
338 * @param elt: xfer element
339 * @param newfd: new value for the fd field
340 * @returns: old value of the fd field
342 #define xfer_element_swap_input_fd(elt, newfd) \
343 xfer_atomic_swap_fd((elt)->xfer, &(elt)->_input_fd, newfd)
344 #define xfer_element_swap_output_fd(elt, newfd) \
345 xfer_atomic_swap_fd((elt)->xfer, &(elt)->_output_fd, newfd)
347 /***********************
348 * XferElement subclasses
350 * These simple subclasses do not introduce any additional public members or
351 * methods, so they do not have their own header file. The functions here
352 * provide their only public interface. The implementation of these elements
353 * can also provide a good prototype for new elements.
356 /* A transfer source that produces LENGTH bytes of random data, for testing
359 * Implemented in source-random.c
361 * @param length: bytes to produce, or zero for no limit
362 * @param prng_seed: initial value for random number generator
363 * @return: new element
365 XferElement *xfer_source_random(guint64 length, guint32 prng_seed);
367 /* Get the ending random seed for the xfer_source_random. Call this after a
368 * transfer has finished, and construct a new xfer_source_random with the seed.
369 * The new source will continue the same random sequence at the next byte. This
370 * is useful for constructing spanned dumps in testing.
372 * @param src: XferSourceRandom object
375 guint32 xfer_source_random_get_seed(XferElement *src);
377 /* A transfer source that produces LENGTH bytes containing repeated
378 * copies of the provided pattern, for testing purposes.
380 * Implemented in source-pattern.c
382 * @param length: bytes to produce, or zero for no limit
383 * @param pattern: Pointer to memory containing the desired byte pattern.
384 * @param pattern_length: Size of pattern to repeat.
385 * @return: new element
387 XferElement *xfer_source_pattern(guint64 length, void * pattern,
388 size_t pattern_length);
390 /* A transfer source that provides bytes read from a file descriptor.
391 * Reading continues until EOF, but the file descriptor is not closed.
393 * Implemented in source-fd.c
395 * @param fd: the file descriptor from which to read
396 * @return: new element
398 XferElement * xfer_source_fd(
401 /* A transfer source that exposes its listening DirectTCPAddrs (via
402 * elt->input_listen_addrs) for external use
404 * Implemented in source-directtcp-listen.c
406 * @return: new element
408 XferElement * xfer_source_directtcp_listen(void);
410 /* A transfer source that connects to a DirectTCP address and pulls data
411 * from it into the transfer.
413 * Implemented in source-directtcp-listen.c
415 * @param addrs: DirectTCP addresses to connect to
416 * @return: new element
418 XferElement * xfer_source_directtcp_connect(DirectTCPAddr *addrs);
420 /* A transfer filter that executes an external application, feeding it data on
421 * stdin and taking the results on stdout.
423 * The memory for ARGV becomes the property of the transfer element and will be
424 * g_strfreev'd when the xfer is destroyed.
426 * Implemented in filter-process.c
428 * @param argv: NULL-terminated command-line arguments
429 * @param need_root: become root before exec'ing the subprocess
430 * @return: new element
432 XferElement *xfer_filter_process(gchar **argv,
435 /* A transfer filter that just applies a bytewise XOR transformation to the data
436 * that passes through it.
438 * Implemented in filter-xor.c
440 * @param xor_key: key for xor operations
441 * @return: new element
443 XferElement *xfer_filter_xor(
444 unsigned char xor_key);
446 /* A transfer destination that consumes all bytes it is given, optionally
447 * validating that they match those produced by source_random
449 * Implemented in dest-null.c
451 * @param prng_seed: if nonzero, validate that the datastream matches
452 * that produced by a random source with this random seed. If zero,
453 * no validation is performed.
454 * @return: new element
456 XferElement *xfer_dest_null(
459 /* A transfer destination that writes bytes to a file descriptor. The file
460 * descriptor is not closed when the transfer is complete.
462 * Implemented in dest-fd.c
464 * @param fd: file descriptor to which to write
465 * @return: new element
467 XferElement *xfer_dest_fd(
470 /* A transfer destination that writes bytes to an in-memory buffer.
472 * Implemented in dest-buffer.c
474 * @param max_size: maximum size for the buffer, or zero for no limit
475 * @return: new element
477 XferElement *xfer_dest_buffer(
480 /* Get the buffer and size from an XferDestBuffer. The resulting buffer
481 * will remain allocated until the XDB itself is freed.
483 * Implemented in dest-buffer.c
485 * @param elt: the element
486 * @param buf (output): buffer pointer
487 * @param size (output): buffer size
489 void xfer_dest_buffer_get(
494 /* A transfer dest that connects to a DirectTCPAddr and sends data to
497 * Implemented in dest-directtcp-connect.c
499 * @param addrs: DirectTCP addresses to connect to
500 * @return: new element
502 XferElement * xfer_dest_directtcp_connect(DirectTCPAddr *addrs);
504 /* A transfer dest that listens for a DirecTCP connection and sends data to it
505 * when connected. Listening addresses are exposed at
506 * elt->output_listen_addrs.
508 * Implemented in dest-directtcp-listen.c
510 * @return: new element
512 XferElement * xfer_dest_directtcp_listen(void);