i#513 drx_buf Part 2: Adds drx_buf code and tests

Constructs a convenience API for initializing and filling three
different kinds of buffers: a fast and a slow circular buffer, and a
trace buffer which notifies the client when the buffer fills.

Review-URL: https://codereview.appspot.com/297700043

api/docs/release.dox

@@ -143,6 +143,8 @@ The changes between version \DR_VERSION and 6.1.0 include:
    drreg_set_bb_properties().
  - Added priority-controlled drmgr_register_thread_init_event_ex() and
    drmgr_register_thread_exit_event_ex().
+ - Added \ref sec_drx_buf to drx: drx_buf_create_circular_buffer(),
+   drx_buf_create_trace_buffer(), and more.
 
 **************************************************
 <hr>

ext/drx/CMakeLists.txt

@@ -39,6 +39,7 @@ set(DynamoRIO_USE_LIBC OFF)
 
 set(srcs
   drx.c
+  drx_buf.c
   # add more here
   )
 

ext/drx/drx.c

@@ -87,6 +87,10 @@ static uint verbose = 0;
     } \
 } while (0)
 
+/* defined in drx_buf.c */
+bool drx_buf_init_library(void);
+void drx_buf_exit_library(void);
+
 /***************************************************************************
  * INIT
  */
@@ -105,7 +109,7 @@ drx_init(void)
     note_base = drmgr_reserve_note_range(DRX_NOTE_COUNT);
     ASSERT(note_base != DRMGR_NOTE_NONE, "failed to reserve note range");
 
-    return true;
+    return drx_buf_init_library();
 }
 
 DR_EXPORT
@@ -119,6 +123,7 @@ drx_exit()
     if (soft_kills_enabled)
         soft_kills_exit();
 
+    drx_buf_exit_library();
     drmgr_exit();
 }
 

ext/drx/drx.dox

@@ -1,5 +1,5 @@
 /* **********************************************************
- * Copyright (c) 2013-2014 Google, Inc.   All rights reserved.
+ * Copyright (c) 2013-2016 Google, Inc.   All rights reserved.
  * **********************************************************/
 
 /*
@@ -71,4 +71,87 @@ parent to simply skip its own termination request.  The nudge handler
 should normally handle multiple requests, as it is not uncommon for the
 parent to kill each child process through multiple mechanisms.
 
+\section sec_drx_buf \p Buffer Filling API
+
+The \p drx library also demonstrates a minimalistic buffer API. Its API is
+currently in flux. These buffers may contain traces of data gathered during
+instrumentation, such as memory traces, instruction traces, etc. Note that
+per-thread buffers are used for all implementations. There currently exist
+three types of buffers.
+
+- \ref sec_drx_buf_trace
+- \ref sec_drx_buf_circular
+- \ref sec_drx_buf_circular_fast
+- \ref sec_drx_buf_api
+- \ref sec_drx_buf_no_api
+
+\section sec_drx_buf_trace Trace Buffer
+
+The trace buffer notifies the user when the buffer fills up and allows the
+client to write the contents to disk or to a pipe, etc. Note that writing
+multiple fields of a struct to the buffer runs the risk of the client being
+notified that the buffer is filled before the entire struct has been written.
+In order to circumvent this limitation, either write the element at the
+highest offset in the struct first, so that the user never sees an
+incompletely-written struct, or if this is not possible, allocate a buffer
+whose size is a multiple of the size of the struct.
+
+\section sec_drx_buf_circular Circular Buffer
+
+This circular buffer will wrap around when it becomes full, and is used
+when a client might only need to remember the most recent portion of a
+sequence of events instead of recording an entire trace of events. This
+circular buffer can be any size, but is specially optimized for a buffer
+size of 65336.
+
+\section sec_drx_buf_circular_fast Fast Circular Buffer
+
+The only special case mentioned in \ref sec_drx_buf_circular is a buffer
+of size 65336. Because the buffer is this size exactly, we can align it
+to a 65336 byte boundary, and increment only the bottom two bytes of the
+base pointer. By this method we are able to wrap around on overflow.
+
+Note that this buffer is very good for homogeneous writes, such as in the
+sample client \p bbuf (see \ref API_samples), where we only write \p app_pc
+sized values. Since the buffer cannot be a different size, when using a
+structure it is a good idea to increment \p buf_ptr to a size that evenly
+divides the size of the buffer.
+
+\section sec_drx_buf_api Using the Buffer API
+
+There is a single API for modifying the buffer which is compatible with each
+of these buffer types. The user must generally load the buffer pointer into
+a register, perform some store operations relative to the register, and then
+finally update the buffer pointer to accommodate these stores. Using offsets
+for subsequent fields in a structure is the most efficient method, but please
+note the warning in \ref sec_drx_buf_trace, where one should either allocate
+an integer multiple of the size of the struct, or always write the last field
+of a struct first.
+
+\code
+/* load the buffer pointer into reg_ptr */
+drx_buf_insert_load_buf_ptr(drcontext, buf, bb, inst, reg_ptr);
+/* Store whatever is in the scratch reg to the buffer at offset 0, and then
+ * to offset 8.
+ */
+drx_buf_insert_buf_store(drcontext, buf, bb, inst, reg_ptr,
+                         DR_REG_NULL, opnd_create_reg(scratch1), OPSZ_PTR, 0);
+drx_buf_insert_buf_store(drcontext, buf, bb, inst, reg_ptr,
+                         DR_REG_NULL, opnd_create_reg(scratch2), OPSZ_PTR, 8);
+/* We wrote 16 bytes (8 bytes of scratch1 and 8 bytes of scratch2), so we
+ * increment the buffer pointer by that amount, using reg_tmp as a temporary
+ * scratch register.
+ */
+drx_buf_insert_update_buf_ptr(drcontext, buf, bb, inst, reg_ptr,
+                              reg_tmp, sizeof(reg_t)*2);
+\endcode
+
+\section sec_drx_buf_no_api Manually Modifying the Buffer
+
+It is possible to manually modify the buffer without calling
+drx_buf_insert_buf_store(). The provided store routines are for convenience
+only, to ensure that an app translation is set for each instruction. If a user
+writes to the buffer without using the provided operations, please make sure an
+app translation is set.
+
 */

ext/drx/drx.h

@@ -258,6 +258,147 @@ drx_open_unique_appid_file(const char *dir, ptr_int_t id,
                            const char *prefix, const char *suffix,
                            uint extra_flags, char *result OUT, size_t result_len);
 
+/***************************************************************************
+ * BUFFER FILLING LIBRARY
+ */
+
+/**
+ * Callback for \p drx_buf_init_trace_buffer(), called when the buffer has
+ * been filled. The valid buffer data is contained within the interval
+ * [buf_base..buf_base+size).
+ */
+typedef void (*drx_buf_full_cb_t)(void *drcontext, void *buf_base, size_t size);
+
+struct _drx_buf_t;
+
+/** Opaque handle which represents a buffer for use by the drx_buf framework. */
+typedef struct _drx_buf_t drx_buf_t;
+
+enum {
+    /**
+     * Buffer size to be specified in drx_buf_create_circular_buffer() in order
+     * to make use of the fast circular buffer optimization.
+     */
+    DRX_BUF_FAST_CIRCULAR_BUFSZ = (1<<16)
+};
+
+/**
+ * Priorities of drmgr instrumentation passes used by drx_buf. Users
+ * of drx_buf can use the names #DRMGR_PRIORITY_NAME_DRX_BUF_INIT and
+ * #DRMGR_PRIORITY_NAME_DRX_BUF_EXIT in the drmgr_priority_t.before field
+ * or can use these numeric priorities in the drmgr_priority_t.priority
+ * field to ensure proper instrumentation pass ordering.
+ */
+enum {
+    /** Priority of drx_buf thread init event */
+    DRMGR_PRIORITY_THREAD_INIT_DRX_BUF       =  -7500,
+    /** Priority of drx_buf thread exit event */
+    DRMGR_PRIORITY_THREAD_EXIT_DRX_BUF       =  -7500,
+};
+
+/** Name of drx_buf thread init priority for buffer initialization. */
+#define DRMGR_PRIORITY_NAME_DRX_BUF_INIT "drx_buf.init"
+
+/** Name of drx_buf thread exit priority for buffer cleanup and full_cb callback. */
+#define DRMGR_PRIORITY_NAME_DRX_BUF_EXIT "drx_buf.exit"
+
+DR_EXPORT
+/**
+ * Initializes the drx_buf extension with a circular buffer which wraps
+ * around when full.
+ *
+ * \note All buffer sizes are supported. However, a buffer size of
+ * #DRX_BUF_FAST_CIRCULAR_BUFSZ bytes is specially optimized for performance.
+ * This buffer is referred to explicitly in the documentation as the "fast
+ * circular buffer".
+ *
+ * \return NULL if unsuccessful, a valid opaque struct pointer if successful.
+ */
+drx_buf_t *
+drx_buf_create_circular_buffer(size_t buf_size);
+
+DR_EXPORT
+/**
+ * Initializes the drx_buf extension with a buffer; \p full_cb is called
+ * when the buffer becomes full.
+ *
+ * \return NULL if unsuccessful, a valid opaque struct pointer if successful.
+ */
+drx_buf_t *
+drx_buf_create_trace_buffer(size_t buffer_size,
+                            drx_buf_full_cb_t full_cb);
+
+DR_EXPORT
+/** Cleans up the buffer associated with \p buf. \returns whether successful. */
+bool
+drx_buf_free(drx_buf_t *buf);
+
+DR_EXPORT
+/**
+ * Inserts instructions to load the address of the TLS buffer at \p where
+ * into \p buf_ptr.
+ */
+void
+drx_buf_insert_load_buf_ptr(void *drcontext, drx_buf_t *buf, instrlist_t *ilist,
+                            instr_t *where, reg_id_t buf_ptr);
+
+DR_EXPORT
+/**
+ * Inserts instructions to increment the buffer pointer by \p stride to accommodate
+ * the writes that occurred since the last time the base pointer was loaded.
+ *
+ * \note \p scratch is only necessary on ARM, in the case of the fast circular
+ * buffer. On x86 scratch is completely unused.
+ */
+void
+drx_buf_insert_update_buf_ptr(void *drcontext, drx_buf_t *buf, instrlist_t *ilist,
+                              instr_t *where, reg_id_t buf_ptr, reg_id_t scratch,
+                              ushort stride);
+
+DR_EXPORT
+/**
+ * Inserts instructions to store \p opsz bytes of \p opnd at \p offset bytes
+ * from \p buf_ptr. \p opnd must be a register or an immediate opnd of some
+ * appropriate size.  \return whether successful.
+ *
+ * \note \p opsz must be either \p OPSZ_1, \p OPSZ_2, \p OPSZ_4 or \p OPSZ_8.
+ *
+ * \note \p scratch is only necessary on ARM when storing an immediate operand.
+ *
+ * \note This method simply wraps a store that also sets an app translation. Make
+ * sure that \p where has a translation set.
+ */
+bool
+drx_buf_insert_buf_store(void *drcontext, drx_buf_t *buf, instrlist_t *ilist,
+                         instr_t *where, reg_id_t buf_ptr, reg_id_t scratch,
+                         opnd_t opnd, opnd_size_t opsz, short offset);
+
+DR_EXPORT
+/**
+ * Retrieves a pointer to the top of the buffer, that is, returns the
+ * same value as would an invocation of drx_buf_insert_load_buf_ptr().
+ */
+void *
+drx_buf_get_buffer_ptr(void *drcontext, drx_buf_t *buf);
+
+DR_EXPORT
+/**
+ * Allows one to set the buffer pointer so that subsequent invocations
+ * of drx_buf_insert_load_buf_ptr() will use this new value instead.
+ */
+void
+drx_buf_set_buffer_ptr(void *drcontext, drx_buf_t *buf, void *new_ptr);
+
+DR_EXPORT
+/** Retrieves a pointer to the base of the buffer. */
+void *
+drx_buf_get_buffer_base(void *drcontext, drx_buf_t *buf);
+
+DR_EXPORT
+/** Retrieves the capacity of the buffer. */
+size_t
+drx_buf_get_buffer_size(void *drcontext, drx_buf_t *buf);
+
 /*@}*/ /* end doxygen group */
 
 #ifdef __cplusplus